Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge master

Browse source
This commit is contained in:
Rajat Venkatesh 2015-09-02 16:33:31 -07:00
commit afd50bc551
271 changed files with 27862 additions and 3857 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
.gitignore vendored
View file

@ -5,6 +5,9 @@
*.war
*.ear
# interpreter
/interpreter/
# conf file
conf/zeppelin-env.sh
conf/zeppelin-site.xml
@ -12,26 +15,30 @@ conf/keystore
conf/truststore
conf/interpreter.json
# other generated files
spark/dependency-reduced-pom.xml
#webapp
zeppelin-web/node_modules
zeppelin-web/dist
zeppelin-web/.tmp
zeppelin-web/.sass-cache
zeppelin-web/bower_components
zeppelin-web/src
**nbproject/
**node/
logs/
run/
metastore_db/
*.log
jobs/
zan-repo/
drivers/
warehouse/
notebook/
local-repo/
# project level
/logs/
/run/
/metastore_db/
/*.log
/jobs/
/zan-repo/
/drivers/
/warehouse/
/notebook/
/local-repo/
**/sessions/
**/data/
@ -73,3 +80,4 @@ auto-save-list
tramp
.\#*
*.swp
**/dependency-reduced-pom.xml

33
.travis.yml
View file

@ -21,20 +21,45 @@ before_install:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
install:
- mvn package -DskipTests -Phadoop-2.3 -Ppyspark -B
before_script:
- mvn package -Pbuild-distr -B
- ./testing/startSparkCluster.sh
-
script:
- mvn verify -Pusing-packaged-distr -B
# spark 1.4
- mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B
- ./testing/startSparkCluster.sh 1.4.0 2.3
- mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B
- ./testing/stopSparkCluster.sh 1.4.0 2.3
# spark 1.3
- rm -rf `pwd`/interpreter/spark
- mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.3.1 2.3
- mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.3.1 2.3
# spark 1.2
- rm -rf `pwd`/interpreter/spark
- mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.2.1 2.3
- mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.2.1 2.3
# spark 1.1
- rm -rf `pwd`/interpreter/spark
- mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.1.1 2.3
- mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.1.1 2.3
after_failure:
- cat target/rat.txt
- cat zeppelin-server/target/rat.txt
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
after_script:
- ./testing/stopSparkCluster.sh
-
notifications:
slack:

44
CONTRIBUTING.md
View file

@ -8,6 +8,9 @@ Contributing to Zeppelin (Source code, Documents, Image, Website) means you agre
3. Contribute your patch via Pull Request.
## SourceControl Workflow
Zeppelin follows [Fork & Pull] (https://github.com/sevntu-checkstyle/sevntu.checkstyle/wiki/Development-workflow-with-Git:-Fork,-Branching,-Commits,-and-Pull-Request) model.
## Setting up
Here are some things you will need to build and test Zeppelin.
@ -17,7 +20,29 @@ Zeppelin uses Git for it's SCM system. Hosted by github.com. `https://github.com
### Integrated Development Environment(IDE)
You are free to use whatever IDE you prefer, or your favorite command line editor.
You are free to use whatever IDE you prefer, or your favorite command line editor.
### Project Structure
Zeppelin project is based on Maven. Maven works by convention & defines [directory structure] (https://maven.apache.org/guides/introduction/introduction-to-the-standard-directory-layout.html) for a project.
The top level pom.xml describes the basic project structure. Currently Zeppelin has the following modules.
<module>zeppelin-interpreter</module>
<module>zeppelin-zengine</module>
<module>spark</module>
<module>markdown</module>
<module>angular</module>
<module>shell</module>
<module>hive</module>
<module>tajo</module>
<module>flink</module>
<module>ignite</module>
<module>lens</module>
<module>cassandra</module>
<module>zeppelin-web</module>
<module>zeppelin-server</module>
<module>zeppelin-distribution</module>
### Code convention
We are following Google Code style:
@ -74,6 +99,23 @@ To build with specific spark / hadoop version
mvn install -Phadoop-2.2 -Dhadoop.version=2.2.0 -Pspark-1.3 -Dspark.version=1.3.0
```
## Tests
Each new File should have its own accompanying unit tests. Each new interpreter should have come with its tests.
Zeppelin has 3 types of tests:
1. Unit Tests: The unit tests run as part of each package's build. E.g SparkInterpeter Module's unit test is SparkInterpreterTest
2. Integration Tests: The intergration tests run after all modules are build. The integration tests launch an instance of Zeppelin server. ZeppelinRestApiTest is an example integration test.
3. GUI integration tests: These tests validate the Zeppelin UI elements. These tests require a running Zepplein server and lauches a web browser to validate Notebook UI elements like Notes and their execution. See ZeppelinIT as an example.
Currently the GUI integration tests are not run in the Maven and are only run in the CI environment when the pull request is submitted to github. Make sure to watch the [CI results] (https://travis-ci.org/apache/incubator-zeppelin/pull_requests) for your pull request.
## Continuous Integration
Zeppelin uses Travis for CI. In the project root there is .travis.yml that configures CI and [publishes CI results] (https://travis-ci.org/apache/incubator-zeppelin/builds)
## Run Zepplin server in development mode
```

37
LICENSE
View file

@ -200,3 +200,40 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
APACHE ZEPPELIN SUBCOMPONENTS:
The Apache Zeppelin project contains subcomponents with separate copyright
notices and license terms. Your use of the source code for the these
subcomponents is subject to the terms and conditions of the following
licenses.
For the PostgreSQL JDBC driver jar file:
Copyright (c) 1997-2011, PostgreSQL Global Development Group
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the PostgreSQL Global Development Group nor the names
of its contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

64
README.md
View file

@ -38,31 +38,57 @@ sudo apt-get install npm
### Build
If you want to build Zeppelin from the source, please first clone this repository. And then:
```
mvn clean package
mvn clean package -DskipTests
```
Build with specific version
Spark 1.1.x
Build with specific Spark version
Spark 1.4.x
```
mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
```
Spark 1.2.x
```
mvn clean package -Pspark-1.2 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
mvn clean package -Pspark-1.4 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
```
Spark 1.3.x
```
mvn clean package -Pspark-1.3 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
```
Spark 1.2.x
```
mvn clean package -Pspark-1.2 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
```
Spark 1.1.x
```
mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
```
CDH 5.X
```
mvn clean package -Pspark-1.2 -Dhadoop.version=2.5.0-cdh5.3.0 -Phadoop-2.4 -DskipTests
```
Yarn (Hadoop 2.2.x and later)
Yarn (Hadoop 2.7.x)
```
mvn clean package -Pspark-1.4 -Dspark.version=1.4.1 -Dhadoop.version=2.7.0 -Phadoop-2.6 -Pyarn -DskipTests
```
Yarn (Hadoop 2.6.x)
```
mvn clean package -Pspark-1.1 -Dhadoop.version=2.6.0 -Phadoop-2.6 -Pyarn -DskipTests
```
Yarn (Hadoop 2.4.x)
```
mvn clean package -Pspark-1.1 -Dhadoop.version=2.4.0 -Phadoop-2.4 -Pyarn -DskipTests
```
Yarn (Hadoop 2.3.x)
```
mvn clean package -Pspark-1.1 -Dhadoop.version=2.3.0 -Phadoop-2.3 -Pyarn -DskipTests
```
Yarn (Hadoop 2.2.x)
```
mvn clean package -Pspark-1.1 -Dhadoop.version=2.2.0 -Phadoop-2.2 -Pyarn -DskipTests
```
Ignite (1.1.0-incubating and later)
```
mvn clean package -Dignite.version=1.1.0-incubating -DskipTests
```
### Configure
If you wish to configure Zeppelin option (like port number), configure the following files:
```
@ -70,7 +96,20 @@ If you wish to configure Zeppelin option (like port number), configure the follo
./conf/zeppelin-site.xml
```
(You can copy ```./conf/zeppelin-env.sh.template``` into ```./conf/zeppelin-env.sh```.
Same for ```zeppein-site.xml```.)
Same for ```zeppelin-site.xml```.)
#### Setting SPARK_HOME and HADOOP_HOME
Without SPARK_HOME and HADOOP_HOME, Zeppelin uses embedded Spark and Hadoop binaries that you have specified with mvn build option.
If you want to use system provided Spark and Hadoop, export SPARK_HOME and HADOOP_HOME in zeppelin-env.sh
You can use any supported version of spark without rebuilding Zeppelin.
```
# ./conf/zeppelin-env.sh
export SPARK_HOME=...
export HADOOP_HOME=...
```
#### External cluster configuration
Mesos
@ -92,7 +131,7 @@ Yarn
### Run
./bin/zeppelin-daemon.sh start
browse localhost:8080 in your browser. 8081 port should be accessible for websocket connection.
browse localhost:8080 in your browser.
For configuration details check __./conf__ subdirectory.
@ -105,7 +144,7 @@ To package final distribution do:
The archive is generated under _zeppelin-distribution/target_ directory
###Run end-to-end tests
Zeppelin comes with a set of end-to-end acceptnce tests driving headless selenium browser
Zeppelin comes with a set of end-to-end acceptance tests driving headless selenium browser
#assumes zeppelin-server running on localhost:8080 (use -Durl=.. to override)
mvn verify
@ -114,4 +153,5 @@ Zeppelin comes with a set of end-to-end acceptnce tests driving headless seleniu
mvn verify -P using-packaged-distr
[![Analytics](https://ga-beacon.appspot.com/UA-45176241-4/apache/incubator-zeppelin/README.md?pixel)](https://github.com/igrigorik/ga-beacon)

15
Roadmap.md
View file

@ -1,17 +1,4 @@
# Zeppelin Roadmap
This document explains the list of planned and work to be done in Zeppelin. It is intended to be exhaustive but some elements might still be missing.
## Features
* Support for multiple separate clusters (one per-notebook)
* Apache Tajo interpreter
* Apache Flink interpreter
* Python interpreter
* Package Zeppelin (RPM,DEB)
* Apache BigTop integration
* Notebook read-only mode
* R interpreter
* Pluggable notebook persistence layer with supporting directory structure
* Printing output with stream
Please check https://cwiki.apache.org/confluence/display/ZEPPELIN/Zeppelin+Roadmap for details

146
_tools/scalastyle.xml Normal file
View file

@ -0,0 +1,146 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!-- NOTE: This was taken and adapted from Apache Spark. -->
<!-- If you wish to turn off checking for a section of code, you can put a comment in the source
before and after the section, with the following syntax: -->
<!-- // scalastyle:off -->
<!-- ... -->
<!-- // naughty stuff -->
<!-- ... -->
<!-- // scalastyle:on -->
<scalastyle>
<name>Scalastyle standard configuration</name>
<check level="error" class="org.scalastyle.file.FileTabChecker" enabled="true"></check>
<!-- <check level="error" class="org.scalastyle.file.FileLengthChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="maxFileLength"><![CDATA[800]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<check level="error" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
<parameters>
<parameter name="header"><![CDATA[/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/]]></parameter>
</parameters>
</check>
<check level="error" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="false"></check>
<check level="error" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.file.FileLineLengthChecker" enabled="true">
<parameters>
<parameter name="maxLineLength"><![CDATA[100]]></parameter>
<parameter name="tabSize"><![CDATA[2]]></parameter>
<parameter name="ignoreImports">true</parameter>
</parameters>
</check>
<check level="error" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true">
<parameters>
<parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
</parameters>
</check>
<check level="error" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true">
<parameters>
<parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
</parameters>
</check>
<check level="error" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true">
<parameters>
<parameter name="regex"><![CDATA[^[a-z][A-Za-z]*$]]></parameter>
</parameters>
</check>
<check level="error" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="false"></check>
<!-- <check level="error" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="illegalImports"><![CDATA[sun._,java.awt._]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<check level="error" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true">
<parameters>
<parameter name="maxParameters"><![CDATA[10]]></parameter>
</parameters>
</check>
<!-- <check level="error" class="org.scalastyle.scalariform.MagicNumberChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="ignore"><![CDATA[-1,0,1,2,3]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<check level="error" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="false"></check>
<check level="error" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="false"></check>
<!-- <check level="error" class="org.scalastyle.scalariform.ReturnChecker" enabled="true"></check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.NullChecker" enabled="true"></check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true"></check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"></check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true"></check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true"></check> -->
<!-- <check level="error" class="org.scalastyle.file.RegexChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="regex"><![CDATA[println]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="maxTypes"><![CDATA[30]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="maximum"><![CDATA[10]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<check level="error" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="false"></check>
<check level="error" class="org.scalastyle.scalariform.IfBraceChecker" enabled="true">
<parameters>
<parameter name="singleLineAllowed"><![CDATA[true]]></parameter>
<parameter name="doubleLineAllowed"><![CDATA[true]]></parameter>
</parameters>
</check>
<!-- <check level="error" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="maxLength"><![CDATA[50]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="regex"><![CDATA[^[a-z][A-Za-z0-9]*$]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true"> -->
<!-- <parameters> -->
<!-- <parameter name="maxMethods"><![CDATA[30]]></parameter> -->
<!-- </parameters> -->
<!-- </check> -->
<!-- <check level="error" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="true"></check> -->
<check level="error" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.file.NoNewLineAtEofChecker" enabled="false"></check>
</scalastyle>

5
angular/pom.xml
View file

@ -22,13 +22,14 @@
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.5.0-incubating-SNAPSHOT</version>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-angular</artifactId>
<packaging>jar</packaging>
<version>0.5.0-incubating-SNAPSHOT</version>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Angular interpreter</name>
<url>http://zeppelin.incubator.apache.org</url>

18
bin/common.sh
View file

@ -45,8 +45,8 @@ if [[ -z "$ZEPPELIN_PID_DIR" ]]; then
fi
if [[ -z "${ZEPPELIN_WAR}" ]]; then
if [[ -d "${ZEPPELIN_HOME}/zeppelin-web/src/main/webapp" ]]; then
export ZEPPELIN_WAR="${ZEPPELIN_HOME}/zeppelin-web/src/main/webapp"
if [[ -d "${ZEPPELIN_HOME}/zeppelin-web/dist" ]]; then
export ZEPPELIN_WAR="${ZEPPELIN_HOME}/zeppelin-web/dist"
else
export ZEPPELIN_WAR=$(find -L "${ZEPPELIN_HOME}" -name "zeppelin-web*.war")
fi
@ -80,22 +80,10 @@ function addEachJarInDir(){
function addJarInDir(){
if [[ -d "${1}" ]]; then
export ZEPPELIN_CLASSPATH="${1}/*:${ZEPPELIN_CLASSPATH}"
ZEPPELIN_CLASSPATH="${1}/*:${ZEPPELIN_CLASSPATH}"
fi
}
if [[ ! -z "${SPARK_HOME}" ]] && [[ -d "${SPARK_HOME}" ]]; then
addJarInDir "${SPARK_HOME}"
fi
if [[ ! -z "${HADOOP_HOME}" ]] && [[ -d "${HADOOP_HOME}" ]]; then
addJarInDir "${HADOOP_HOME}"
fi
if [[ ! -z "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
fi
export ZEPPELIN_CLASSPATH
# Text encoding for

74
bin/interpreter.sh
View file

@ -57,9 +57,6 @@ fi
addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
addJarInDir "${INTERPRETER_DIR}"
export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
HOSTNAME=$(hostname)
ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
@ -73,6 +70,77 @@ if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
$(mkdir -p "${ZEPPELIN_LOG_DIR}")
fi
# set spark related env variables
if [[ "${INTERPRETER_ID}" == "spark" ]]; then
# add Hadoop jars into classpath
if [[ -n "${HADOOP_HOME}" ]]; then
# Apache
addEachJarInDir "${HADOOP_HOME}/share"
# CDH
addJarInDir "${HADOOP_HOME}"
addJarInDir "${HADOOP_HOME}/lib"
fi
# autodetect HADOOP_CONF_HOME by heuristic
if [[ -n "${HADOOP_HOME}" ]] && [[ -z "${HADOOP_CONF_DIR}" ]]; then
if [[ -d "${HADOOP_HOME}/etc/hadoop" ]]; then
export HADOOP_CONF_DIR="${HADOOP_HOME}/etc/hadoop"
elif [[ -d "/etc/hadoop/conf" ]]; then
export HADOOP_CONF_DIR="/etc/hadoop/conf"
fi
fi
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
fi
# add Spark jars into classpath
if [[ -n "${SPARK_HOME}" ]]; then
addJarInDir "${SPARK_HOME}/lib"
PYSPARKPATH="${SPARK_HOME}/python:${SPARK_HOME}/python/lib/pyspark.zip:${SPARK_HOME}/python/lib/py4j-0.8.2.1-src.zip"
else
addJarInDir "${INTERPRETER_DIR}/dep"
PYSPARKPATH="${ZEPPELIN_HOME}/interpreter/spark/pyspark/pyspark.zip:${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-0.8.2.1-src.zip"
fi
# autodetect SPARK_CONF_DIR
if [[ -n "${SPARK_HOME}" ]] && [[ -z "${SPARK_CONF_DIR}" ]]; then
if [[ -d "${SPARK_HOME}/conf" ]]; then
SPARK_CONF_DIR="${SPARK_HOME}/conf"
fi
fi
# read spark-*.conf if exists
if [[ -d "${SPARK_CONF_DIR}" ]]; then
ls ${SPARK_CONF_DIR}/spark-*.conf > /dev/null 2>&1
if [[ "$?" -eq 0 ]]; then
for file in ${SPARK_CONF_DIR}/spark-*.conf; do
while read -r line; do
echo "${line}" | grep -e "^spark[.]" > /dev/null
if [ "$?" -ne 0 ]; then
# skip the line not started with 'spark.'
continue;
fi
SPARK_CONF_KEY=`echo "${line}" | sed -e 's/\(^spark[^ ]*\)[ \t]*\(.*\)/\1/g'`
SPARK_CONF_VALUE=`echo "${line}" | sed -e 's/\(^spark[^ ]*\)[ \t]*\(.*\)/\2/g'`
export ZEPPELIN_JAVA_OPTS+=" -D${SPARK_CONF_KEY}=\"${SPARK_CONF_VALUE}\""
done < "${file}"
done
fi
fi
if [[ -z "${PYTHONPATH}" ]]; then
export PYTHONPATH="${PYSPARKPATH}"
else
export PYTHONPATH="${PYTHONPATH}:${PYSPARKPATH}"
fi
unset PYSPARKPATH
fi
export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} -cp ${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} &
pid=$!

20
bin/zeppelin-daemon.sh
View file

@ -101,19 +101,27 @@ function wait_for_zeppelin_to_die() {
local pid
local count
pid=$1
timeout=$2
count=0
while [[ "${count}" -lt 10 ]]; do
timeoutTime=$(date "+%s")
let "timeoutTime+=$timeout"
currentTime=$(date "+%s")
forceKill=1
while [[ $currentTime -lt $timeoutTime ]]; do
$(kill ${pid} > /dev/null 2> /dev/null)
if kill -0 ${pid} > /dev/null 2>&1; then
sleep 3
let "count+=1"
else
forceKill=0
break
fi
if [[ "${count}" == "5" ]]; then
currentTime=$(date "+%s")
done
if [[ forceKill -ne 0 ]]; then
$(kill -9 ${pid} > /dev/null 2> /dev/null)
fi
done
}
function wait_zeppelin_is_up_for_ci() {
@ -187,7 +195,7 @@ function stop() {
if [[ -z "${pid}" ]]; then
echo "${ZEPPELIN_NAME} is not running"
else
wait_for_zeppelin_to_die $pid
wait_for_zeppelin_to_die $pid 40
$(rm -f ${ZEPPELIN_PID})
action_msg "${ZEPPELIN_NAME} stop" "${SET_OK}"
fi
@ -200,7 +208,7 @@ function stop() {
fi
pid=$(cat ${f})
wait_for_zeppelin_to_die $pid
wait_for_zeppelin_to_die $pid 20
$(rm -f ${f})
done

15
bin/zeppelin.sh
View file

@ -21,10 +21,7 @@
# Run Zeppelin
#
function usage() {
echo "Usage: bin/zeppelin.sh [--config <conf-dir>] [spark options] [application options]"
exit 0
}
USAGE="Usage: bin/zeppelin.sh [--config <conf-dir>]"
if [[ "$1" == "--config" ]]; then
shift
@ -88,12 +85,4 @@ if [[ ! -d "${ZEPPELIN_NOTEBOOK_DIR}" ]]; then
$(mkdir -p "${ZEPPELIN_NOTEBOOK_DIR}")
fi
if [[ ! -z "${SPARK_HOME}" ]]; then
source "${SPARK_HOME}/bin/utils.sh"
SUBMIT_USAGE_FUNCTION=usage
gatherSparkSubmitOpts "$@"
ZEPPELIN_RUNNER="${SPARK_HOME}/bin/spark-submit"
$(exec $ZEPPELIN_NICENESS $ZEPPELIN_RUNNER --class $ZEPPELIN_SERVER "${SUBMISSION_OPTS[@]}" --driver-java-options -Dzeppelin.log.file=$ZEPPELIN_LOGFILE spark-shell "${APPLICATION_OPTS[@]}")
else
$(exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_SERVER "$@")
fi
$(exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_SERVER "$@")

299
cassandra/pom.xml Normal file
View file

@ -0,0 +1,299 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-cassandra</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Cassandra</name>
<description>Zeppelin cassandra support</description>
<url>http://zeppelin.incubator.apache.org</url>
<properties>
<cassandra.driver.version>2.1.7.1</cassandra.driver.version>
<snappy.version>1.0.5.4</snappy.version>
<lz4.version>1.2.0</lz4.version>
<scala.version>2.11.7</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<commons-lang.version>3.3.2</commons-lang.version>
<scalate.version>1.7.1</scalate.version>
<!--TEST-->
<scalatest.version>2.2.4</scalatest.version>
<junit.version>4.12</junit.version>
<achilles.version>3.2.2</achilles.version>
<assertj.version>1.7.0</assertj.version>
<mockito.version>1.9.5</mockito.version>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-core</artifactId>
<version>${cassandra.driver.version}</version>
</dependency>
<!-- Compression libraries for the cassandra-driver protocol. -->
<!-- Include both compression options to make to simplify deployment. -->
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>
<groupId>net.jpountz.lz4</groupId>
<artifactId>lz4</artifactId>
<version>${lz4.version}</version>
</dependency>
<!-- End of compression libraries -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang.version}</version>
</dependency>
<dependency>
<groupId>org.scalatra.scalate</groupId>
<artifactId>scalate-core_${scala.binary.version}</artifactId>
<version>${scalate.version}</version>
</dependency>
<!--TEST-->
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>${scalatest.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>info.archinnov</groupId>
<artifactId>achilles-junit</artifactId>
<version>${achilles.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>16.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>${assertj.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!-- Plugin to compile Scala code -->
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<goals>
<goal>compile</goal>
</goals>
<phase>compile</phase>
</execution>
<execution>
<id>test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
<phase>test-compile</phase>
</execution>
<execution>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>1.0</version>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalatra.scalate</groupId>
<artifactId>maven-scalate-plugin_${scala.binary.version}</artifactId>
<version>1.7.1</version>
<executions>
<execution>
<id>compile</id>
<phase>process-classes</phase>
<goals>
<goal>precompile</goal>
</goals>
<configuration>
<resourcesSourceDirectory>${basedir}/src/main/resources/scalate</resourcesSourceDirectory>
<contextClass>org.fusesource.scalate.DefaultRenderContext</contextClass>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/cassandra</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/cassandra</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

339
cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java Normal file
View file

@ -0,0 +1,339 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.ProtocolOptions.Compression;
import com.datastax.driver.core.Session;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import static com.datastax.driver.core.ProtocolOptions.DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS;
import static java.lang.Integer.parseInt;
/**
* Interpreter for Apache Cassandra CQL query language
*/
public class CassandraInterpreter extends Interpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(CassandraInterpreter.class);
public static final String CASSANDRA_INTERPRETER_PARALLELISM = "cassandra.interpreter" +
".parallelism";
public static final String CASSANDRA_HOSTS = "cassandra.hosts";
public static final String CASSANDRA_PORT = "cassandra.native.port";
public static final String CASSANDRA_PROTOCOL_VERSION = "cassandra.protocol.version";
public static final String CASSANDRA_CLUSTER_NAME = "cassandra.cluster";
public static final String CASSANDRA_KEYSPACE_NAME = "cassandra.keyspace";
public static final String CASSANDRA_COMPRESSION_PROTOCOL = "cassandra.compression.protocol";
public static final String CASSANDRA_CREDENTIALS_USERNAME = "cassandra.credentials.username";
public static final String CASSANDRA_CREDENTIALS_PASSWORD = "cassandra.credentials.password";
public static final String CASSANDRA_LOAD_BALANCING_POLICY = "cassandra.load.balancing.policy";
public static final String CASSANDRA_RETRY_POLICY = "cassandra.retry.policy";
public static final String CASSANDRA_RECONNECTION_POLICY = "cassandra.reconnection.policy";
public static final String CASSANDRA_SPECULATIVE_EXECUTION_POLICY =
"cassandra.speculative.execution.policy";
public static final String CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS =
"cassandra.max.schema.agreement.wait.second";
public static final String CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL =
"cassandra.pooling.new.connection.threshold.local";
public static final String CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE =
"cassandra.pooling.new.connection.threshold.remote";
public static final String CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL =
"cassandra.pooling.max.connection.per.host.local";
public static final String CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE =
"cassandra.pooling.max.connection.per.host.remote";
public static final String CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL =
"cassandra.pooling.core.connection.per.host.local";
public static final String CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE =
"cassandra.pooling.core.connection.per.host.remote";
public static final String CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL =
"cassandra.pooling.max.request.per.connection.local";
public static final String CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE =
"cassandra.pooling.max.request.per.connection.remote";
public static final String CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS =
"cassandra.pooling.idle.timeout.seconds";
public static final String CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS =
"cassandra.pooling.pool.timeout.millisecs";
public static final String CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS =
"cassandra.pooling.heartbeat.interval.seconds";
public static final String CASSANDRA_QUERY_DEFAULT_CONSISTENCY =
"cassandra.query.default.consistency";
public static final String CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY =
"cassandra.query.default.serial.consistency";
public static final String CASSANDRA_QUERY_DEFAULT_FETCH_SIZE =
"cassandra.query.default.fetchSize";
public static final String CASSANDRA_QUERY_DEFAULT_IDEMPOTENCE =
"cassandra.query.default.idempotence";
public static final String CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS =
"cassandra.socket.connection.timeout.millisecs";
public static final String CASSANDRA_SOCKET_KEEP_ALIVE =
"cassandra.socket.keep.alive";
public static final String CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS =
"cassandra.socket.read.timeout.millisecs";
public static final String CASSANDRA_SOCKET_RECEIVED_BUFFER_SIZE_BYTES =
"cassandra.socket.received.buffer.size.bytes";
public static final String CASSANDRA_SOCKET_REUSE_ADDRESS =
"cassandra.socket.reuse.address";
public static final String CASSANDRA_SOCKET_SEND_BUFFER_SIZE_BYTES =
"cassandra.socket.send.buffer.size.bytes";
public static final String CASSANDRA_SOCKET_SO_LINGER =
"cassandra.socket.soLinger";
public static final String CASSANDRA_SOCKET_TCP_NO_DELAY =
"cassandra.socket.tcp.no_delay";
public static final String DEFAULT_HOST = "localhost";
public static final String DEFAULT_PORT = "9042";
public static final String DEFAULT_CLUSTER = "Test Cluster";
public static final String DEFAULT_KEYSPACE = "system";
public static final String DEFAULT_PROTOCOL_VERSION = "3";
public static final String DEFAULT_COMPRESSION = "NONE";
public static final String DEFAULT_CREDENTIAL = "none";
public static final String DEFAULT_POLICY = "DEFAULT";
public static final String DEFAULT_PARALLELISM = "10";
static String DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100";
static String DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "100";
static String DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2";
static String DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1";
static String DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8";
static String DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2";
static String DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024";
static String DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256";
public static final String DEFAULT_IDLE_TIMEOUT = "120";
public static final String DEFAULT_POOL_TIMEOUT = "5000";
public static final String DEFAULT_HEARTBEAT_INTERVAL = "30";
public static final String DEFAULT_CONSISTENCY = "ONE";
public static final String DEFAULT_SERIAL_CONSISTENCY = "SERIAL";
public static final String DEFAULT_FETCH_SIZE = "5000";
public static final String DEFAULT_CONNECTION_TIMEOUT = "5000";
public static final String DEFAULT_READ_TIMEOUT = "12000";
public static final String DEFAULT_TCP_NO_DELAY = "true";
public static final String DOWNGRADING_CONSISTENCY_RETRY = "DOWNGRADING_CONSISTENCY";
public static final String FALLTHROUGH_RETRY = "FALLTHROUGH";
public static final String LOGGING_DEFAULT_RETRY = "LOGGING_DEFAULT";
public static final String LOGGING_DOWNGRADING_RETRY = "LOGGING_DOWNGRADING";
public static final String LOGGING_FALLTHROUGH_RETRY = "LOGGING_FALLTHROUGH";
public static final List<String> NO_COMPLETION = new ArrayList<>();
InterpreterLogic helper;
Cluster cluster;
Session session;
private JavaDriverConfig driverConfig = new JavaDriverConfig();
public CassandraInterpreter(Properties properties) {
super(properties);
}
static {
LOGGER.info("Bootstrapping Cassandra Interpreter");
Interpreter.register("cassandra", "cassandra", CassandraInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(CASSANDRA_HOSTS, DEFAULT_HOST,
"Comma separated Cassandra hosts (DNS name or " +
"IP address). Default = localhost. Ex: '192.168.0.12,node2,node3'")
.add(CASSANDRA_PORT, DEFAULT_PORT, "Cassandra native port. Default = 9042")
.add(CASSANDRA_PROTOCOL_VERSION, DEFAULT_PROTOCOL_VERSION,
"Cassandra protocol version. Default = 3")
.add(CASSANDRA_CLUSTER_NAME, DEFAULT_CLUSTER, "Cassandra cluster name. " +
"Default = 'Test Cluster'")
.add(CASSANDRA_KEYSPACE_NAME, DEFAULT_KEYSPACE, "Cassandra keyspace name. " +
"Default = 'system'")
.add(CASSANDRA_COMPRESSION_PROTOCOL, DEFAULT_COMPRESSION,
"Cassandra compression protocol. " +
"Available values: NONE, SNAPPY, LZ4. Default = NONE")
.add(CASSANDRA_CREDENTIALS_USERNAME, DEFAULT_CREDENTIAL,
"Cassandra credentials username. " +
"Default = 'none'")
.add(CASSANDRA_CREDENTIALS_PASSWORD, DEFAULT_CREDENTIAL,
"Cassandra credentials password. " +
"Default = 'none'")
.add(CASSANDRA_LOAD_BALANCING_POLICY, DEFAULT_POLICY, "Cassandra Load Balancing Policy. " +
"Default = new TokenAwarePolicy(new DCAwareRoundRobinPolicy())")
.add(CASSANDRA_RETRY_POLICY, DEFAULT_POLICY, "Cassandra Retry Policy. " +
"Default = DefaultRetryPolicy.INSTANCE")
.add(CASSANDRA_RECONNECTION_POLICY, DEFAULT_POLICY, "Cassandra Reconnection Policy. " +
"Default = new ExponentialReconnectionPolicy(1000, 10 * 60 * 1000)")
.add(CASSANDRA_SPECULATIVE_EXECUTION_POLICY, DEFAULT_POLICY,
"Cassandra Speculative Execution Policy. " +
"Default = NoSpeculativeExecutionPolicy.INSTANCE")
.add(CASSANDRA_INTERPRETER_PARALLELISM, DEFAULT_PARALLELISM,
"Cassandra interpreter parallelism" +
".Default = 10")
.add(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS,
DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS + ""
, "Cassandra max schema agreement wait in second" +
".Default = ProtocolOptions.DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS")
.add(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL,
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL,
"Cassandra new connection threshold local. " +
"Protocol V2 and below default = 100" +
"Protocol V3 and above default = 800")
.add(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE,
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE,
"Cassandra new connection threshold remove. " +
"Protocol V2 and below default = 100" +
"Protocol V3 and above default = 200")
.add(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL,
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL,
"Cassandra core connection per host local. " +
"Protocol V2 and below default = 2" +
"Protocol V3 and above default = 1")
.add(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE,
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE,
"Cassandra core connection per host remove. " +
"Protocol V2 and below default = 1" +
"Protocol V3 and above default = 1")
.add(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL,
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL,
"Cassandra max connection per host local. " +
"Protocol V2 and below default = 8" +
"Protocol V3 and above default = 1")
.add(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE,
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE,
"Cassandra max connection per host remote. " +
"Protocol V2 and below default = 2" +
"Protocol V3 and above default = 1")
.add(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL,
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL,
"Cassandra max request per connection local. " +
"Protocol V2 and below default = 128" +
"Protocol V3 and above default = 1024")
.add(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE,
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE,
"Cassandra max request per connection remote. " +
"Protocol V2 and below default = 128" +
"Protocol V3 and above default = 256")
.add(CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS, DEFAULT_IDLE_TIMEOUT,
"Cassandra idle time out in seconds. Default = 120")
.add(CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS, DEFAULT_POOL_TIMEOUT,
"Cassandra pool time out in millisecs. Default = 5000")
.add(CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS, DEFAULT_HEARTBEAT_INTERVAL,
"Cassandra pool heartbeat interval in secs. Default = 30")
.add(CASSANDRA_QUERY_DEFAULT_CONSISTENCY, DEFAULT_CONSISTENCY,
"Cassandra query default consistency level. Default = ONE")
.add(CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY, DEFAULT_SERIAL_CONSISTENCY,
"Cassandra query default serial consistency level. Default = SERIAL")
.add(CASSANDRA_QUERY_DEFAULT_FETCH_SIZE, DEFAULT_FETCH_SIZE,
"Cassandra query default fetch size. Default = 5000")
.add(CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS, DEFAULT_CONNECTION_TIMEOUT,
"Cassandra socket default connection timeout in millisecs. Default = 5000")
.add(CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS, DEFAULT_READ_TIMEOUT,
"Cassandra socket read timeout in millisecs. Default = 12000")
.add(CASSANDRA_SOCKET_TCP_NO_DELAY, DEFAULT_TCP_NO_DELAY,
"Cassandra socket TCP no delay. Default = true")
.build());
}
@Override
public void open() {
final String[] addresses = getProperty(CASSANDRA_HOSTS).split(",");
final int port = parseInt(getProperty(CASSANDRA_PORT));
StringBuilder hosts = new StringBuilder();
for (String address : addresses) {
hosts.append(address).append(",");
}
LOGGER.info("Bootstrapping Cassandra Java Driver to connect to " + hosts.toString() +
"on port " + port);
Compression compression = driverConfig.getCompressionProtocol(this);
cluster = Cluster.builder()
.addContactPoints(addresses)
.withPort(port)
.withProtocolVersion(driverConfig.getProtocolVersion(this))
.withClusterName(getProperty(CASSANDRA_CLUSTER_NAME))
.withCompression(compression)
.withCredentials(getProperty(CASSANDRA_CREDENTIALS_USERNAME),
getProperty(CASSANDRA_CREDENTIALS_PASSWORD))
.withLoadBalancingPolicy(driverConfig.getLoadBalancingPolicy(this))
.withRetryPolicy(driverConfig.getRetryPolicy(this))
.withReconnectionPolicy(driverConfig.getReconnectionPolicy(this))
.withSpeculativeExecutionPolicy(driverConfig.getSpeculativeExecutionPolicy(this))
.withMaxSchemaAgreementWaitSeconds(
parseInt(getProperty(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS)))
.withPoolingOptions(driverConfig.getPoolingOptions(this))
.withQueryOptions(driverConfig.getQueryOptions(this))
.withSocketOptions(driverConfig.getSocketOptions(this))
.build();
session = cluster.connect();
helper = new InterpreterLogic(session);
}
@Override
public void close() {
session.close();
cluster.close();
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return helper.interpret(session, st, context);
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public List<String> completion(String buf, int cursor) {
return NO_COMPLETION;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton()
.createOrGetParallelScheduler(CassandraInterpreter.class.getName() + this.hashCode(),
parseInt(getProperty(CASSANDRA_INTERPRETER_PARALLELISM)));
}
@Override
public void destroy() {
super.destroy();
this.close();
}
}

26
cassandra/src/main/java/org/apache/zeppelin/cassandra/ParsingException.java Normal file
View file

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra;
/**
* Parsing Exception for Cassandra CQL statement
*/
public class ParsingException extends RuntimeException{
public ParsingException(String message) {
super(message);
}
}

69
cassandra/src/main/resources/scalate/allTables.ssp Normal file
View file

@ -0,0 +1,69 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
#import(java.util.UUID)
<%@ val allTables: Map[(UUID,String),List[String]] %>
<div class="container">
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for (((ksId,ksName), tables) <- allTables)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${ksId}" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;${ksName}</span>
</a>
</h4>
</div>
<div id="${ksId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
#if (tables.nonEmpty)
<table class="table">
<thead>
<tr><th>Tables</th></tr>
</thead>
<tbody>
#for (table <- tables)
<tr class="text-primary"><td>${table}</td></tr>
#end
</tbody>
</table>
#else
<span><h4>No Table</h4></span>
#end
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
#end
</div>
</div>
</div>

54
cassandra/src/main/resources/scalate/clusterContent.ssp Normal file
View file

@ -0,0 +1,54 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val clusterContent: ClusterContent %>
<div class="container">
<!-- Cluster -->
${unescape(clusterContent.clusterDetails)}
<div class="row"></div>
<!-- Keyspaces -->
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-danger">Keyspaces</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for ((id, name, keyspaceHTML) <- clusterContent.keyspaces)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;${name}</span>
</a>
</h4>
</div>
<div id="${id}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
${unescape(keyspaceHTML)}
</div>
</div>
</div>
#end
</div>
</div>
</div>

44
cassandra/src/main/resources/scalate/clusterDetails.ssp Normal file
View file

@ -0,0 +1,44 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val clusterDetails: ClusterDetails %>
<div class="row">
<div class="col-md-4"></div>
<div class="col-md-4 col-offset-md-4">
<div class="table-responsive table-bordered">
<table class="table">
<caption>
<h4 class="text-muted">
<i class="glyphicon glyphicon-dashboard"/>&nbsp;${clusterDetails.name}
</h4>
</caption>
<thead>
<tr>
<th>Partitioner</th>
</tr>
</thead>
<tbody>
<tr>
<td>${clusterDetails.partitioner}</td>
</tr>
<tbody>
</table>
</div>
</div>
</div>

36
cassandra/src/main/resources/scalate/dropDownMenuForCluster.ssp Normal file
View file

@ -0,0 +1,36 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val clusterContent: ClusterContent %>
<li role="presentation" class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-muted"><i class="glyphicon glyphicon-dashboard"/>&nbsp;<strong>${clusterContent.clusterName}</strong></span>
<span class="text-muted caret"></span>
<ul class="dropdown-menu">
<li class="dropdown-header"><span class="text-danger">Keyspaces</span></li>
#for((id, name, _) <- clusterContent.keyspaces)
<li>
<a role="button" data-toggle="collapse" data-target="#${id}">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;${name}</span>
</a>
</li>
#end
</ul>
</a>
</li>

52
cassandra/src/main/resources/scalate/dropDownMenuForKeyspace.ssp Normal file
View file

@ -0,0 +1,52 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val ksContent: KeyspaceContent %>
<li role="presentation" class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;<strong>${ksContent.keyspaceName}</strong></span>
<span class="text-danger caret"></span>
<ul class="dropdown-menu">
#if(ksContent.tables.nonEmpty)
<li class="dropdown-header"><span class="text-primary">Tables</span></li>
#for((id,name,_) <- ksContent.tables)
<li>
<a role="button" data-toggle="collapse" data-target="#${id}">
<span class="text-primary"><i class="glyphicon glyphicon-th-list"/>&nbsp;${name}</span>
</a>
</li>
#end
#end
#if(ksContent.udts.nonEmpty)
<li role="separator" class="divider text-muted"></li>
<li class="dropdown-header"><span class="text-warning">User Defined Types</span></li>
#for((id,name,_) <- ksContent.udts)
<li>
<a role="button" data-toggle="collapse" data-target="#${id}">
<span class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;${name}</span>
</a>
</li>
#end
#end
</ul>
</a>
</li>

901
cassandra/src/main/resources/scalate/helpMenu.ssp Normal file
View file

@ -0,0 +1,901 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(java.util.UUID)
#import(com.datastax.driver.core.utils.UUIDs)
<%@ val basicCommandsId: UUID = UUIDs.random() %>
<%@ val schemaDiscoveryId: UUID = UUIDs.random() %>
<%@ val queryParamsId: UUID = UUIDs.random() %>
<%@ val preparedStatementsId: UUID = UUIDs.random() %>
<%@ val dynamicFormsId: UUID = UUIDs.random() %>
<%@ val configurationId: UUID = UUIDs.random() %>
<%@ val miscId: UUID = UUIDs.random() %>
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li role="presentation" class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-info"><i class="glyphicon glyphicon-book"/>&nbsp;<strong>Please select ...</strong></span>
<span class="text-info caret"></span>
<ul class="dropdown-menu">
<li class="dropdown-header"><span class="text-info">Topics</span></li>
<li>
<a role="button" data-toggle="collapse" data-target="#${basicCommandsId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Basic Commands</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${schemaDiscoveryId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Schema Discovery</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${queryParamsId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Query Parameters</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${preparedStatementsId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Prepared Statements</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${dynamicFormsId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Dynamic Forms</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${configurationId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Interpreter Configuration</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#${miscId}">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Misc</span>
</a>
</li>
</ul>
</a>
</li>
<li>
<a><span class="text-info"><strong>CASSANDRA INTERPRETER DOCUMENTATION</strong></span></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-info"><strong>About ...</strong></span>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<span class="text-info">Version <strong>1.0</strong></span>
</a>
</li>
<li>
<a role="button">
<span class="text-info">Java Driver Version <strong>2.1.7.1</strong></span>
</a>
</li>
<li>
<a role="button">
<span class="text-info">Author <strong>@doanduyhai</strong></span>
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</nav>
<br/><br/>
<div class="container">
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${basicCommandsId}" aria-expanded="false">
<span class="text-info"><strong>Basic Commands</strong></span>
</a>
</h4>
</div>
<div id="${basicCommandsId}" class="panel-collapse collapse in" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<h3>I CQL Statements</h3>
<p>This interpreter is compatible with any CQL statement supported by Cassandra. Ex:
<br/><br/>
<div class="row">
<div class="col-md-6 col-md-offset-3">
<pre>
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
SELECT * FROM users WHERE login='jdoe';
</pre>
</div>
</div>
<br/>
Each statement should be separated by a <strong>semi-colon</strong> (;).
<br/>
<strong>Multi-line</strong> statements as well as multiple statements on the <strong>same line</strong>
are also supported as long as they are separated by a semi-colon. Ex:
<br/>
<br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
USE spark_demo;
SELECT * FROM albums_by_country LIMIT 1; SELECT * FROM countries LIMIT 1;
SELECT *
FROM artists
WHERE login='jlennon';
</pre>
</div>
</div>
<br/>
<strong>Batch</strong> statements are supported and can span multiple lines, as well as
<strong>DDL</strong>(CREATE/ALTER/DROP) statements:
<br/>
<br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
BEGIN BATCH
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
INSERT INTO users_preferences(login,account_type) VALUES('jdoe','BASIC');
APPLY BATCH;
CREATE TABLE IF NOT EXISTS test(
key int PRIMARY KEY,
value text
);
</pre>
</div>
</div>
<br/>
CQL statements are <strong>case-insensitive</strong> (except for column names and values).
This means that the following statements are equivalent and valid:
<br/>
<br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
Insert into users(login,name) vAlues('hsue','Helen SUE');
</pre>
</div>
</div>
<br/>
The complete list of all CQL statements and versions can be found below:
<br/><br/>
<div class="row">
<div class="col-md-6 col-md-offset-3">
<table class="table table-bordered">
<thead>
<tr><th>Cassandra version</th><th>Documentation</th></tr>
</thead>
<tbody>
<tr>
<td><strong>2.2</strong></td>
<td>
<a href="http://docs.datastax.com/en/cql/3.3/cql/cqlIntro.html" target="_blank">
http://docs.datastax.com/en/cql/3.3/cql/cqlIntro.html
</a>
</td>
</tr>
<tr>
<td><strong>2.1 & 2.0</strong></td>
<td>
<a href="http://docs.datastax.com/en/cql/3.1/cql/cql_intro_c.html" target="_blank">
http://docs.datastax.com/en/cql/3.1/cql/cql_intro_c.html
</a>
</td>
</tr>
<tr>
<td><strong>1.2</strong></td>
<td>
<a href="http://docs.datastax.com/en/cql/3.0/cql/aboutCQL.html" target="_blank">
http://docs.datastax.com/en/cql/3.0/cql/aboutCQL.html
</a>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</p>
<h3>II Comments</h3>
<p>
It is possible to add comments between statements. Single line comments start with the
<strong>hash</strong> sign (#). Multi-line comments are enclosed between
<strong>&sol;&ast;&ast;</strong> and <strong>&ast;&ast;&sol;</strong>. Ex:
<br/>
<br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
#First comment
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
/**
Multi line
comments
**/
Insert into users(login,name) vAlues('hsue','Helen SUE');
</pre>
</div>
</div>
<br/>
</p>
<h3>III Syntax Validation</h3>
<p>
The interpreters is shipped with a <em>built-in syntax validator</em>. This validator only
checks for <strong>basic syntax errors</strong>. All CQL-related syntax validation is delegated
directly to <strong>Cassandra</strong>
<br/><br/>
Most of the time, syntax errors are due to missing semi-colons between statements or typo errors.
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${schemaDiscoveryId}" aria-expanded="false">
<span class="text-info"><strong>Schema Discovery</strong></span>
</a>
</h4>
</div>
<div id="${schemaDiscoveryId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<h3>I Commands For Discovery</h3>
<p>
To make schema discovery easier and more interactive, the following commands are supported:
<br/><br/>
<table class="table table-bordered">
<thead>
<tr><th>Command</th><th>Description</th></tr>
</thead>
<tbody>
<tr>
<td><strong>DESCRIBE CLUSTER;</strong></td>
<td>Show the current cluster name and its partitioner</td>
</tr>
<tr>
<td><strong>DESCRIBE KEYSPACES;</strong></td>
<td>List all existing keyspaces in the cluster and their configuration
(replication factor, durable write ...)</td>
</tr>
<tr>
<td><strong>DESCRIBE TABLES;</strong></td>
<td>List all existing keyspaces in the cluster and for each, all the tables name</td>
</tr>
<tr>
<td><strong>DESCRIBE KEYSPACE &lt;keyspace name&gt;;</strong></td>
<td>Describe the given keyspace configuration and all its table details (name, columns, ...)</td>
</tr>
<tr>
<td><strong>DESCRIBE TABLE <em>(&lt;keyspace name&gt;).</em>&lt;table name&gt;;</strong></td>
<td>
Describe the given table. If the keyspace is not provided, the current
<strong>logged in</strong> keyspace is used. If there is no logged in keyspace,
the default <em>system</em> keyspace is used. If no table is found, an error message is raised
</td>
</tr>
<tr>
<td><strong>DESCRIBE TYPE <em>(&lt;keyspace name&gt;).</em>&lt;type name&gt;;</strong></td>
<td>
Describe the given type(UDT). If the keyspace is not provided, the current
<strong>logged in</strong> keyspace is used. If there is no logged in keyspace,
the default <em>system</em> keyspace is used. If no type is found, an error message is raised
</td>
</tr>
</tbody>
</table>
<br/>
<div class="alert alert-danger" role="alert">
Please note that each <strong>DESCRIBE</strong> command should be ended by <strong>a semi-colon</strong>.
</div>
</p>
<h3>II Schema Display</h3>
<p>
The schema objects (cluster, keyspace, table &amp; type) are displayed in a tabular format.
There is a <strong>drop-down</strong> menu on the top left corner to expand objects details.
On the top right menu is shown the Icon legend.
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${queryParamsId}" aria-expanded="false">
<span class="text-info"><strong>Query Parameters</strong></span>
</a>
</h4>
</div>
<div id="${queryParamsId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<p>
Sometimes you want to be able to pass runtime query parameters to your statements.
Those parameters are <strong>not</strong> part of the CQL specs and are specific to the interpreter.
Below is the list of all parameters:
<br/><br/>
<table class="table table-bordered">
<caption>
<h4>Query Parameters</h4>
</caption>
<thead>
<tr>
<th>Parameter</th>
<th>Syntax</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>Consistency Level</td>
<td><strong>@consistency=<em>value</em></strong></td>
<td>Apply the given consistency level to all queries in the paragraph</td>
</tr>
<tr>
<td>Serial Consistency Level</td>
<td><strong>@serialConsistency=<em>value</em></strong></td>
<td>Apply the given serial consistency level to all queries in the paragraph</td>
</tr>
<tr>
<td>Timestamp</td>
<td><strong>@timestamp=<em>long value</em></strong></td>
<td>Apply the given timestamp to all queries in the paragraph.<br/>
Please note that timestamp value passed directly in CQL statement will override this value
</td>
</tr>
<tr>
<td>Retry Policy</td>
<td><strong>@retryPolicy=<em>value</em></strong></td>
<td>Apply the given retry policy to all queries in the paragraph</td>
</tr>
<tr>
<td>Fetch Size</td>
<td><strong>@fetchSize=<em>int value</em></strong></td>
<td>Apply the given fetch size to all queries in the paragraph</td>
</tr>
</tbody>
</table>
<br/>
Some parameters only accept restricted values:
<br/><br/>
<table class="table table-bordered">
<caption>
<h4>Allowed Values</h4>
</caption>
<thead>
<tr>
<th>Parameter</th>
<th>Possible Values</th>
</tr>
</thead>
<tbody>
<tr>
<td>Consistency Level</td>
<td><strong>ALL, ANY, ONE, TWO, THREE, QUORUM, LOCAL_ONE, LOCAL_QUORUM, EACH_QUORUM</strong></td>
</tr>
<tr>
<td>Serial Consistency Level</td>
<td><strong>SERIAL, LOCAL_SERIAL</strong></td>
</tr>
<tr>
<td>Timestamp</td>
<td>Any long value</td>
</tr>
<tr>
<td>Retry Policy</td>
<td>
<strong>
DEFAULT, DOWNGRADING_CONSISTENCY, FALLTHROUGH, LOGGING_DEFAULT,
LOGGING_DOWNGRADING, LOGGING_FALLTHROUGH
</strong>
</td>
</tr>
<tr>
<td>Fetch Size</td>
<td>Any integer value</td>
</tr>
</tbody>
</table>
<br/>
<div class="alert alert-danger" role="alert">
Please note that you <strong>should not add semi-colon (;)</strong> at the end of each parameter statement
</div>
Some example:
<br/><br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
CREATE TABLE IF NOT EXISTS spark_demo.ts(
key int PRIMARY KEY,
value text
);
TRUNCATE spark_demo.ts;
# Timestamp in the past
@timestamp=10
# Force timestamp directly in the first insert
INSERT INTO spark_demo.ts(key,value) VALUES(1,'first insert') USING TIMESTAMP 100;
# Select some data to make the clock turn
SELECT * FROM spark_demo.albums LIMIT 100;
# Now insert using the timestamp parameter set at the beginning(10)
INSERT INTO spark_demo.ts(key,value) VALUES(1,'second insert');
# Check for the result. You should see 'first insert'
SELECT value FROM spark_demo.ts WHERE key=1;
</pre>
</div>
</div>
<br/>
Some remarks about query parameters:
<br/><br/>
<div class="alert alert-info" role="alert">
<ul>
<li><strong>many</strong> query parameters can be set in the same paragraph</li>
<li>if the <strong>same</strong> query parameter is set many time with different values,
the interpreter only take into account the first value
</li>
<li>each query parameter applies to <strong>all</strong> CQL statement in the same paragraph,
unless you override the option using plain CQL text (like forcing timestamp with the USING clause)
</li>
<li>the order of each query parameter with regard to CQL statement does not matter</li>
</ul>
</div>
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${preparedStatementsId}" aria-expanded="false">
<span class="text-info"><strong>Prepared Statements</strong></span>
</a>
</h4>
</div>
<div id="${preparedStatementsId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<h3>I Syntax</h3>
<br/>
<p>
For performance reason, it is better to <strong>prepare statements</strong> before-hand and reuse
them later by providing bound values. This interpreter provides 3 commands to handle prepared and
bound statements:
<br/><br/>
<ol>
<li><strong>@prepare</strong></li>
<li><strong>@bind</strong></li>
<li><strong>@remove_prepared</strong></li>
</ol>
<br/>
Example:
<br/>
<div class="row">
<div class="col-md-10 col-md-offset-1">
<pre>
@prepare[statement_name]=...
@bind[statement_name]=’text’, 1223, ’2015-07-30 12:00:01’, null, true, [‘list_item1’, ’list_item2’]
@bind[statement_name_with_no_bound_value]
@remove_prepare[statement_name]
</pre>
</div>
</div>
<br/>
<h3>II @prepare</h3>
<br/>
<p>
You can use the syntax "<strong>@prepare[statement_name]=SELECT ...</strong>" to create a prepared statement.
The <em>statement_name</em> is mandatory because the interpreter prepares the given statement with the
Java driver and saves the generated prepared statement in an internal map, using the provided
<em>statement_name</em> as search key.
<br/><br/>
<div class="alert alert-info">
Please note that this internal prepared statement map is shared with <strong>all notebooks</strong>
and <strong>all paragraphs</strong> because there is only one instance of the interpreter for Cassandra
</div>
<br/>
<div class="alert alert-warning">
If the interpreter encounters many @prepare for the <strong>same statement_name</strong> (key),
only the <strong>first</strong> statement will be taken into account.
</div>
<br/>
Example:
<br/>
<div class="row">
<div class="col-md-10 col-md-offset-1">
<pre>
@prepare[select]=SELECT * FROM spark_demo.albums LIMIT ?
@prepare[select]=SELECT * FROM spark_demo.artists LIMIT ?
</pre>
</div>
</div>
<br/>
For the above example, the prepared statement is <strong>"SELECT * FROM spark_demo.albums LIMIT ?"</strong>.
<em>"SELECT * FROM spark_demo.artists LIMIT ?"</em> is ignored because an entry already exists in the
prepared statements map with the key <strong>select</strong>.
<br/><br/>
In the context of Zeppelin, a notebook can be scheduled to be executed at regular interval,
thus it is necessary to avoid re-preparing many time the same statement (considered an anti-pattern).
</p>
<h3>III @bind</h3>
<br/>
<p>
Once the statement is prepared (possibly in a separated notebook/paragraph). You can bind values to it:
<br/><br/>
<div class="row">
<div class="col-md-10 col-md-offset-1">
<pre>
@bind[select_first]=10
</pre>
</div>
</div>
<br/>
Bound values are not mandatory for the <strong>@bind</strong> statement.
However if you provide bound values, they need to comply to some syntax:
<ul>
<li>String values should be enclosed between simple quotes ( ‘ )</li>
<li>Date values should be enclosed between simple quotes ( ‘ ) and respect the formats:
<ol>
<li>yyyy-MM-dd HH:MM:ss</li>
<li>yyyy-MM-dd HH:MM:ss.SSS</li>
</ol>
</li>
<li><strong>null</strong> is parsed as-is</li>
<li><strong>boolean</strong> (true|false) are parsed as-is </li>
<li>collection values must follow the
<a href="http://docs.datastax.com/en/cql/3.1/cql/cql_using/use_collections_c.html" target="_blank">standard CQL syntax</a>:
<ul>
<li>list: [‘list_item1’, ’list_item2’, ...]</li>
<li>set: {‘set_item1’, ‘set_item2’, …}</li>
<li>map: {‘key1’: ‘val1’, ‘key2’: ‘val2’, …}</li>
</ul>
</li>
<li>
tuple values should be enclosed between parenthesis
(see <a href="http://docs.datastax.com/en/cql/3.1/cql/cql_reference/tupleType.html" target="_blank">tuple CQL syntax</a>):
(‘text’, 123, true)
</li>
<li>
udt values should be enclosed between brackets
(see <a href="http://docs.datastax.com/en/cql/3.1/cql/cql_using/cqlUseUDT.html" target="_blank">udt CQL syntax</a>):
{stree_name: ‘Beverly Hills’, number: 104, zip_code: 90020, state: ‘California’, …}
</li>
</ul>
<br/>
<div class="alert alert-info">
It is possible to use the <strong>@bind</strong> statement inside a batch: <br/>
<pre>
BEGIN BATCH
@bind[insert_user]='jdoe','John DOE'
UPDATE users SET age = 27 WHERE login='hsue';
APPLY BATCH;
</pre>
</div>
<br/>
</p>
<h3>IV @remove_prepare</h3>
<br/>
<p>
To avoid for a prepared statement to stay forever in the prepared statement map, you can use the <strong>@remove_prepare[statement_name]</strong> syntax
to remove it. Removing a non-existing prepared statement yields no error.
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${dynamicFormsId}" aria-expanded="false">
<span class="text-info"><strong>Dynamic Forms</strong></span>
</a>
</h4>
</div>
<div id="${dynamicFormsId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<p>
Instead of hard-coding your CQL queries, it is possible to use the mustache syntax (<strong>{{ }}</strong>)
to inject simple value or multiple choices forms.
<br/><br/>
The syntax for simple parameter is: <strong>{{input_Label=default value}}</strong>.
The default value is mandatory because the first time the paragraph is executed,
we launch the CQL query before rendering the form so at least one value should be provided.
<br/><br/>
The syntax for multiple choices parameter is: <strong>{{input_Label=value1 | value2 | … | valueN }}</strong>.
By default the first choice is used for CQL query the first time the paragraph is executed.
<br/><br/>
Example:
<br/>
<div class="row">
<div class="col-md-10 col-md-offset-1">
<pre>
#Secondary index on performer style
SELECT name, country, performer
FROM spark_demo.performers
WHERE name='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}'
AND styles CONTAINS '{{style=Rock}}';
</pre>
</div>
</div>
<br/>
In the above example, the first CQL query will be executed for <em>performer='Sheryl Crow'</em>
AND <em>style='Rock'</em>. For subsequent queries, you can change the value directly using the form.
Please note that we enclosed the {{ }} block between simple quotes (') because Cassandra expects a String here.
We could have also use the <strong>{{style='Rock'}}</strong> syntax but this time, the value
displayed on the form is <em>'Rock'</em> and not <em>Rock</em>.
<br/><br/>
<div class="alert alert-info">
It is also possible to use dynamic forms for <strong>prepared statements</strong>: <br/>
<strong>@bind[select]=='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}', '{{style=Rock}}'</strong>
</div>
</pre>
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${configurationId}" aria-expanded="false">
<span class="text-info"><strong>Interpreter Configuration</strong></span>
</a>
</h4>
</div>
<div id="${configurationId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
The <strong>Cassandra</strong> interpreter comes with some some configuration values for the Java driver:
<table class="table table-bordered">
<caption>
<h4>Interpreter Configuration</h4>
</caption>
<thead>
<tr>
<th>Parameter</th>
<th>Default Value</th>
</tr>
</thead>
<tbody>
<tr>
<td>cassandra.cluster</td>
<td><strong>Test Cluster</strong></td>
</tr>
<tr>
<td>cassandra.compression.protocol</td>
<td><strong>NONE</strong>, possible values: LZ4, SNAPPY</td>
</tr>
<tr>
<td>cassandra.credentials.password</td>
<td><strong>none</strong></td>
</tr>
<tr>
<td>cassandra.credentials.username</td>
<td><strong>none</strong></td>
</tr>
<tr>
<td>cassandra.hosts</td>
<td><strong>localhost</strong></td>
</tr>
<tr>
<td>cassandra.interpreter.parallelism</td>
<td><strong>10</strong></td>
</tr>
<tr>
<td>cassandra.keyspace</td>
<td><strong>system</strong></td>
</tr>
<tr>
<td>cassandra.load.balancing.policy</td>
<td><strong>DEFAULT</strong>, or a FQCN of a custom class</td>
</tr>
<tr>
<td>cassandra.max.schema.agreement.wait.second</td>
<td><strong>10</strong></td>
</tr>
<tr>
<td>cassandra.native.port</td>
<td><strong>9042</strong></td>
</tr>
<tr>
<td>cassandra.pooling.core.connection.per.host.local</td>
<td><strong>Protocol V2 and below: 2, V3 and above: 1</strong></td>
</tr>
<tr>
<td>cassandra.pooling.core.connection.per.host.remote</td>
<td><strong>Protocol V2 and below: 1, V3 and above: 1</strong></td>
</tr>
<tr>
<td>cassandra.pooling.heartbeat.interval.seconds</td>
<td><strong>30</strong></td>
</tr>
<tr>
<td>cassandra.pooling.idle.timeout.seconds</td>
<td><strong>Test Cluster</strong></td>
</tr>
<tr>
<td>cassandra.pooling.max.connection.per.host.local</td>
<td><strong>Protocol V2 and below: 8, V3 and above: 1</strong></td>
</tr>
<tr>
<td>cassandra.pooling.max.connection.per.host.remote</td>
<td><strong>Protocol V2 and below: 2, V3 and above: 1</strong></td>
</tr>
<tr>
<td>cassandra.pooling.max.request.per.connection.local</td>
<td><strong>Protocol V2 and below: 128, V3 and above: 1024</strong></td>
</tr>
<tr>
<td>cassandra.pooling.max.request.per.connection.remote</td>
<td><strong>Protocol V2 and below: 128, V3 and above: 256</strong></td>
</tr>
<tr>
<td>cassandra.pooling.new.connection.threshold.local</td>
<td><strong>Protocol V2 and below: 100, V3 and above: 800</strong></td>
</tr>
<tr>
<td>cassandra.pooling.new.connection.threshold.remote</td>
<td><strong>Protocol V2 and below: 100, V3 and above: 200</strong></td>
</tr>
<tr>
<td>cassandra.pooling.pool.timeout.millisecs</td>
<td><strong>5000</strong></td>
</tr>
<tr>
<td>cassandra.protocol.version</td>
<td><strong>3</strong></td>
</tr>
<tr>
<td>cassandra.query.default.consistency</td>
<td><strong>ONE</strong></td>
</tr>
<tr>
<td>cassandra.query.default.fetchSize</td>
<td><strong>5000</strong></td>
</tr>
<tr>
<td>cassandra.query.default.serial.consistency</td>
<td><strong>SERIAL</strong></td>
</tr>
<tr>
<td>cassandra.reconnection.policy</td>
<td><strong>DEFAULT</strong>, or a FQCN of a custom class</td>
</tr>
<tr>
<td>cassandra.retry.policy</td>
<td><strong>DEFAULT</strong>, or a FQCN of a custom class</td>
</tr>
<tr>
<td>cassandra.socket.connection.timeout.millisecs</td>
<td><strong>500</strong></td>
</tr>
<tr>
<td>cassandra.socket.read.timeout.millisecs</td>
<td><strong>12000</strong></td>
</tr>
<tr>
<td>cassandra.socket.tcp.no_delay</td>
<td><strong>true</strong></td>
</tr>
<tr>
<td>cassandra.speculative.execution.policy</td>
<td><strong>DEFAULT</strong>, or a FQCN of a custom class</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${miscId}" aria-expanded="false">
<span class="text-info"><strong>Miscellaneous</strong></span>
</a>
</h4>
</div>
<div id="${miscId}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<h3>Execution parallelism</h3>
It is possible to execute many paragraphs in parallel. However, at the back-end side, we’re still using <strong>synchronous</strong> queries. Asynchronous execution is only possible when it is possible to return a Future value in the <strong>InterpreterResult</strong>. It may be an interesting proposal for the Zeppelin project.
</div>
</div>
</div>
</div>

91
cassandra/src/main/resources/scalate/keyspaceContent.ssp Normal file
View file

@ -0,0 +1,91 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val ksContent: KeyspaceContent %>
<div class="container">
<!-- Keyspace -->
${unescape(ksContent.keyspaceDetails)}
<!-- Tables -->
<div class="row"></div>
#if (ksContent.tables.nonEmpty)
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-primary">Tables</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for((id,name,tableHTML) <- ksContent.tables)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false">
<span class="text-primary">
<i class="glyphicon glyphicon-th-list"/>&nbsp;${name}
</span>
</a>
</h4>
</div>
<div id="${id}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
${unescape(tableHTML)}
</div>
</div>
</div>
#end
</div>
</div>
#end
#if (ksContent.udts.nonEmpty)
<!-- UDTs -->
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-warning">User Defined Types</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
#for((id,name,udtHTML) <- ksContent.udts)
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#${id}" aria-expanded="false">
<span class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;${name}</span>
</a>
</h4>
</div>
<div id="${id}" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
${unescape(udtHTML)}
</div>
</div>
</div>
#end
</div>
</div>
#end
</div>

61
cassandra/src/main/resources/scalate/keyspaceDetails.ssp Normal file
View file

@ -0,0 +1,61 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
#import(scala.util.parsing.json.JSONObject)
<%@ val ksDetails: KeyspaceDetails %>
<%@ val withCaption: Boolean%>
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
#if (withCaption)
<caption>
<h4 class="text-danger">
<i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;${ksDetails.name}
</h4>
</caption>
#end
<thead>
<tr>
<th class="col-md-10">Replication</th>
<th class="col-md-2">Durable Writes</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-10">${ksDetails.getReplicationMap}</td>
<td class="col-md-2">${ksDetails.durableWrites}</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${ksDetails.uniqueId}_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="${ksDetails.uniqueId}_asCQL">
<pre class="well">${ksDetails.asCQL}</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>

94
cassandra/src/main/resources/scalate/menu.ssp Normal file
View file

@ -0,0 +1,94 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
<%@ val statement: String %>
<%@ val dropDownMenu: String = "" %>
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
${unescape(dropDownMenu)}
<li>
<a><strong>${statement}</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>

24
cassandra/src/main/resources/scalate/noResult.ssp Normal file
View file

@ -0,0 +1,24 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
<div class="container">
<div class="row text-center">
<h4>No Result</h4>
</div>
<br>
</div>

65
cassandra/src/main/resources/scalate/noResultWithExecutionInfo.ssp Normal file
View file

@ -0,0 +1,65 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
<%@ val query: String%>
<%@ val consistency: String%>
<%@ val triedHosts: String%>
<%@ val queriedHosts: String%>
<%@ val schemaInAgreement: String%>
<div class="container">
<div class="row text-center">
<h4>No Result</h4>
</div>
<br/>
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3 table-responsive table-bordered">
<table class="table">
<caption><h5>Last query execution info</h5></caption>
<thead>
<tr>
<th>Info</th>
<th>Value</th>
</tr>
</thead>
<tbody>
<tr>
<td>Statement</td>
<td>${query}</td>
</tr>
<tr>
<td>Achieved Consistency</td>
<td>${consistency}</td>
</tr>
<tr>
<td>Tried Hosts</td>
<td>${triedHosts}</td>
</tr>
<tr>
<td>Queried Hosts</td>
<td>${queriedHosts}</td>
</tr>
<tr>
<td>Schema In Agreement</td>
<td>${schemaInAgreement}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>

147
cassandra/src/main/resources/scalate/tableDetails.ssp Normal file
View file

@ -0,0 +1,147 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val tableDetails: TableDetails %>
<%@ val withCaption: Boolean%>
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
#if(withCaption)
<caption><h4 class="text-primary"><i class="glyphicon glyphicon-th-list"/>&nbsp;${tableDetails.tableName}</h4></caption>
#end
<thead>
<tr>
<th class="col-md-4">Column Type</th>
<th class="col-md-4">Column Name</th>
<th class="col-md-4">Data Type</th>
</tr>
</thead>
<tbody>
#for (column <- tableDetails.columns)
#match (column.columnType)
#case(PartitionKey)
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
#match (column.index)
#case (Some(index))
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#case(StaticColumn)
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
#match (column.index)
#case (Some(index))
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#case(ClusteringColumn(ASC))
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes" title="Sort ASC"/>
#match (column.index)
#case (Some(index))
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#case(ClusteringColumn(DESC))
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes-alt" title="Sort DESC"/>
#match (column.index)
#case (Some(index))
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#otherwise
<tr>
<td class="col-md-4">
#match (column.index)
#case (Some(index))
<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>${index.name}</em> <strong>${index.info}</strong>
#case (None)
<span></span>
#end
</td>
<td class="col-md-4">${column.name}</td>
<td class="col-md-4">${column.dataType}</td>
</tr>
#end
#end
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${tableDetails.uniqueId}_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="${tableDetails.uniqueId}_asCQL">
<pre class="well">${tableDetails.asCQL}</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>

61
cassandra/src/main/resources/scalate/udtDetails.ssp Normal file
View file

@ -0,0 +1,61 @@
<%--
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
--%>
#import(org.apache.zeppelin.cassandra.MetaDataHierarchy._)
<%@ val udtDetails: UDTDetails %>
<%@ val withCaption: Boolean%>
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
#if(withCaption)
<caption><h4 class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;${udtDetails.typeName}</h4></caption>
#end
<thead>
<tr>
<th class="col-md-6">Column Name</th>
<th class="col-md-6">Data Type</th>
</tr>
</thead>
<tbody>
#for (column <- udtDetails.columns )
<tr>
<td class="col-md-6">${column.name}</td>
<td class="col-md-6">${column.dataType}</td>
</tr>
#end
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#${udtDetails.uniqueId}_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="${udtDetails.uniqueId}_asCQL">
<pre class="well">${udtDetails.asCQL}</pre>
</div>
</div>
</div>
</div>
<div class="col-md-3"></div>
</div>

65
cassandra/src/main/scala/org/apache/zeppelin/cassandra/BoundValuesParser.scala Normal file
View file

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import java.text.SimpleDateFormat
import java.util.{Date}
import scala.util.parsing.combinator._
/**
* Parser of bound values passed into @bind parameters
*/
class BoundValuesParser extends RegexParsers with JavaTokenParsers {
val STANDARD_DATE_PATTERN = """(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})""".r
val ACCURATE_DATE_PATTERN = """(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})""".r
def value : Parser[String] = "null" | "true" | "false" | zeppelinVariable |
map | list | set | tuple| udt |
decimal | integer | standardDate | quotedString
def integer: Parser[String] = """\d+""".r ^^{_.toLong.toString}
def decimal: Parser[String] = """[+-]?(?:(?:\d+\.(?:\d*)?)|(?:\.\d+))""".r ^^{_.toDouble.toString}
def standardDate: Parser[String] = s"""'${STANDARD_DATE_PATTERN.toString}(?:\\.\\d{3})?'""".r ^^{_.replaceAll("'","")}
def quotedString: Parser[String] = """'[^']+'""".r //^^ {_.replaceAll("(?<!')'","")}
def list: Parser[String] = "["~>repsep(value, ",")<~"]" ^^ {_.mkString("[",",","]")}
def set: Parser[String] = "{"~>repsep(value, ",")<~"}" ^^ {_.mkString("{",",","}")}
def map: Parser[String] = "{"~>repsep(member, ",")<~"}" ^^{_.mkString("{",", ","}")}
def tuple: Parser[String] = "(" ~> repsep(value, ",") <~ ")" ^^{_.mkString("(",",",")")}
def udt: Parser[String] = "{"~>repsep(udtMember, ",")<~"}" ^^{_.mkString("{",", ","}")}
def member: Parser[String] = quotedString ~ ":" ~ value ^^{ case name~sep~mapVal => name+": "+mapVal}
def udtColumnName: Parser[String] = """(?:(?:[a-zA-Z][a-zA-Z0-9_]*)|(?:"[^"]+"))""".r
def udtMember: Parser[String] = udtColumnName ~ ":" ~ value ^^{ case name~sep~mapVal => name+": "+mapVal}
def zeppelinVariable: Parser[String] = "{{"~"""\w+=[^}]+""".r~"}}" ^^{case prefix~variable~suffix => prefix+variable+suffix}
def values: Parser[List[String]] = repsep(value, ",")
}

298
cassandra/src/main/scala/org/apache/zeppelin/cassandra/DisplaySystem.scala Normal file
View file

@ -0,0 +1,298 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import java.util.UUID
import com.datastax.driver.core.ColumnMetadata.IndexMetadata
import com.datastax.driver.core.utils.UUIDs
import org.apache.zeppelin.cassandra.MetaDataHierarchy._
import org.fusesource.scalate.TemplateEngine
import scala.collection.JavaConverters._
import com.datastax.driver.core._
import scala.collection.immutable.ListMap
/**
* Format and display
* schema meta data
*/
object DisplaySystem {
val engine = new TemplateEngine
val CLUSTER_DETAILS_TEMPLATE = "scalate/clusterDetails.ssp"
val KEYSPACE_DETAILS_TEMPLATE = "scalate/keyspaceDetails.ssp"
val TABLE_DETAILS_TEMPLATE = "scalate/tableDetails.ssp"
val UDT_DETAILS_TEMPLATE = "scalate/udtDetails.ssp"
val MENU_TEMPLATE = "scalate/menu.ssp"
val CLUSTER_DROPDOWN_TEMPLATE = "scalate/dropDownMenuForCluster.ssp"
val KEYSPACE_DROPDOWN_TEMPLATE = "scalate/dropDownMenuForKeyspace.ssp"
val CLUSTER_CONTENT_TEMPLATE = "scalate/clusterContent.ssp"
val KEYSPACE_CONTENT_TEMPLATE = "scalate/keyspaceContent.ssp"
val ALL_TABLES_TEMPLATE = "scalate/allTables.ssp"
object TableDisplay {
def format(statement: String, meta: TableMetadata, withCaption: Boolean): String = {
MenuDisplay.formatMenu(statement) + formatWithoutMenu(meta, withCaption)
}
protected[DisplaySystem] def formatWithoutMenu(meta: TableMetadata, withCaption: Boolean): String = {
val tableName: String = meta.getName
val columnsDetails = MetaDataConverter.tableMetaToColumnDetails(meta)
engine.layout(TABLE_DETAILS_TEMPLATE,
Map[String, Any]("tableDetails" -> TableDetails(tableName, columnsDetails, meta.exportAsString), "withCaption" -> withCaption))
}
}
object UDTDisplay {
def format(statement: String, userType: UserType, withCaption: Boolean): String = {
MenuDisplay.formatMenu(statement) ++ formatWithoutMenu(userType, withCaption)
}
protected[DisplaySystem] def formatWithoutMenu(userType: UserType, withCaption: Boolean): String = {
val udtName: String = userType.getTypeName
val columnsDetails = MetaDataConverter.userTypeToColumnDetails(userType)
engine.layout(UDT_DETAILS_TEMPLATE,
Map[String, Any]("udtDetails" -> UDTDetails(udtName, columnsDetails, userType.exportAsString), "withCaption" -> withCaption))
}
}
object KeyspaceDisplay {
private def formatCQLQuery(cql: String): String = {
cql.replaceAll(""" WITH REPLICATION = \{"""," WITH REPLICATION = \\{")
.replaceAll("('[^']+'\\s*:\\s+'[^']+',?)","\n\t$1")
.replaceAll(""" \} AND DURABLE_WRITES = """," \\}\nAND DURABLE_WRITES = ")
}
protected[cassandra] def formatKeyspaceOnly(meta: KeyspaceMetadata, withCaption: Boolean): String = {
val ksDetails = KeyspaceDetails(meta.getName,
meta.getReplication.asScala.toMap,
meta.isDurableWrites,
formatCQLQuery(meta.asCQLQuery()))
engine.layout(KEYSPACE_DETAILS_TEMPLATE,
Map[String, Any]("ksDetails" -> ksDetails, "withCaption" -> withCaption))
}
def formatKeyspaceContent(statement: String, meta: KeyspaceMetadata): String = {
val ksName: String = meta.getName
val ksDetails = formatKeyspaceOnly(meta, false)
val tableDetails: List[(UUID, String, String)] = meta.getTables.asScala.toList
.sortBy(meta => meta.getName)
.map(meta => (UUIDs.timeBased(), meta.getName, TableDisplay.formatWithoutMenu(meta, false)))
val udtDetails: List[(UUID, String, String)] = meta.getUserTypes.asScala.toList
.sortBy(udt => udt.getTypeName)
.map(udt => (UUIDs.timeBased(), udt.getTypeName, UDTDisplay.formatWithoutMenu(udt, false)))
val ksContent: KeyspaceContent = KeyspaceContent(ksName, ksDetails, tableDetails, udtDetails)
MenuDisplay.formatMenuForKeyspace(statement, ksContent) +
engine.layout(KEYSPACE_CONTENT_TEMPLATE,
Map[String, Any]("statement" -> statement, "ksContent" -> ksContent))
}
}
object ClusterDisplay {
def formatClusterOnly(statement: String, meta: Metadata, withMenu: Boolean = true): String = {
val clusterDetails: ClusterDetails = ClusterDetails(meta.getClusterName, meta.getPartitioner)
val content: String = engine.layout(CLUSTER_DETAILS_TEMPLATE,
Map[String, Any]("clusterDetails" -> clusterDetails))
if(withMenu) MenuDisplay.formatMenu(statement) + content else content
}
def formatClusterContent(statement: String, meta: Metadata): String = {
val clusterName: String = meta.getClusterName
val clusterDetails: String = formatClusterOnly(statement, meta, false)
val keyspaceDetails: List[(UUID, String, String)] = meta.getKeyspaces.asScala.toList
.sortBy(ks => ks.getName)
.map(ks => (UUIDs.timeBased(), ks.getName, KeyspaceDisplay.formatKeyspaceOnly(ks, false)))
val clusterContent: ClusterContent = ClusterContent(clusterName, clusterDetails, keyspaceDetails)
MenuDisplay.formatMenuForCluster(statement, clusterContent) +
engine.layout(CLUSTER_CONTENT_TEMPLATE,
Map[String, Any]("clusterContent" -> clusterContent))
}
def formatAllTables(statement: String, meta: Metadata): String = {
val ksMetas: List[KeyspaceMetadata] = meta.getKeyspaces.asScala.toList
.sortBy(ks => ks.getName)
val allTables: Map[(UUID, String), List[String]] = ListMap.empty ++
ksMetas
.map(ks => {
((UUIDs.timeBased(), ks.getName),
ks.getTables.asScala.toList.map(table => table.getName).sortBy(name => name))
})
.sortBy{case ((id,name), _) => name}
val keyspaceDetails: List[(UUID, String, String)] = allTables
.keySet.toList.sortBy{case(id,ksName) => ksName}
.map{case(id,ksName) => (id,ksName, "")}
val clusterContent: ClusterContent = ClusterContent(meta.getClusterName, "", keyspaceDetails)
MenuDisplay.formatMenuForCluster(statement, clusterContent) +
engine.layout(ALL_TABLES_TEMPLATE,
Map[String, Any]("allTables" -> allTables))
}
}
object HelpDisplay {
def formatHelp(): String = {
engine.layout("/scalate/helpMenu.ssp")
}
}
object NoResultDisplay {
val formatNoResult: String = engine.layout("/scalate/noResult.ssp")
def noResultWithExecutionInfo(lastQuery: String, execInfo: ExecutionInfo): String = {
val consistency = Option(execInfo.getAchievedConsistencyLevel).getOrElse("N/A")
val queriedHosts = execInfo.getQueriedHost.toString.replaceAll("/","").replaceAll("""\[""","").replaceAll("""\]""","")
val triedHosts = execInfo.getTriedHosts.toString.replaceAll("/","").replaceAll("""\[""","").replaceAll("""\]""","")
val schemaInAgreement = Option(execInfo.isSchemaInAgreement).map(_.toString).getOrElse("N/A")
engine.layout("/scalate/noResultWithExecutionInfo.ssp",
Map[String,Any]("query" -> lastQuery, "consistency" -> consistency,
"triedHosts" -> triedHosts, "queriedHosts" -> queriedHosts,
"schemaInAgreement" -> schemaInAgreement))
}
}
private object MenuDisplay {
def formatMenu(statement: String, dropDownMenu: String = ""): String = {
engine.layout(MENU_TEMPLATE,
Map[String, Any]("statement" -> statement, "dropDownMenu" -> dropDownMenu))
}
def formatMenuForKeyspace(statement: String, ksContent: KeyspaceContent): String = {
val dropDownMenu: String = engine.layout(KEYSPACE_DROPDOWN_TEMPLATE,
Map[String, Any]("ksContent" -> ksContent))
formatMenu(statement, dropDownMenu)
}
def formatMenuForCluster(statement: String, clusterContent: ClusterContent): String = {
val dropDownMenu: String = engine.layout(CLUSTER_DROPDOWN_TEMPLATE,
Map[String, Any]("clusterContent" -> clusterContent))
formatMenu(statement, dropDownMenu)
}
}
}
class ColumnMetaWrapper(val columnMeta: ColumnMetadata) {
def canEqual(other: Any): Boolean = other.isInstanceOf[ColumnMetaWrapper]
override def equals(other: Any): Boolean = other match {
case that: ColumnMetaWrapper => (that canEqual this) &&
(columnMeta.getName == that.columnMeta.getName)
case _ => false
}
override def hashCode: Int = columnMeta.getName.hashCode
}
/**
* Convert Java driver
* meta data structure
* to our own structure
*/
object MetaDataConverter {
def tableMetaToColumnDetails(meta: TableMetadata): List[ColumnDetails] = {
val partitionKeys: List[ColumnMetaWrapper] = meta.getPartitionKey.asScala.toList.map(new ColumnMetaWrapper(_))
val clusteringColumns: List[ColumnMetaWrapper] = meta.getClusteringColumns.asScala.toList.map(new ColumnMetaWrapper(_))
val columns: List[ColumnMetaWrapper] = meta.getColumns.asScala.toList.map(new ColumnMetaWrapper(_))
.diff(partitionKeys).diff(clusteringColumns)
val clusteringOrders = meta.getClusteringOrder.asScala.toList
convertPartitionKeys(partitionKeys):::
extractStaticColumns(columns):::
convertClusteringColumns(clusteringColumns, clusteringOrders):::
extractNormalColumns(columns)
}
def userTypeToColumnDetails(userType: UserType): List[ColumnDetails] = {
userType.getFieldNames.asScala.toList
.map(name => new ColumnDetails(name, NormalColumn, userType.getFieldType(name), None))
}
private def extractNormalColumns(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.filter(_.columnMeta.isStatic == false)
.map(c => new ColumnDetails(c.columnMeta.getName, NormalColumn, c.columnMeta.getType, extractIndexDetail(c)))
}
private def extractIndexDetail(column: ColumnMetaWrapper): Option[IndexDetails] = {
val indexOption = Option(column.columnMeta.getIndex)
def buildIndexInfo(indexMeta: IndexMetadata): String = {
if(indexMeta.isKeys) "KEYS"
if(indexMeta.isEntries) "ENTRIES"
if(indexMeta.isFull) "FULL"
if(indexMeta.isCustomIndex) s"Class = ${indexMeta.getIndexClassName}"
else ""
}
indexOption.map(index => IndexDetails(index.getName, buildIndexInfo(index)))
}
private def extractStaticColumns(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.filter(_.columnMeta.isStatic == true)
.map(c => new ColumnDetails(c.columnMeta.getName, StaticColumn, c.columnMeta.getType, extractIndexDetail(c)))
}
private def convertClusteringColumns(columns: List[ColumnMetaWrapper], orders: List[TableMetadata.Order]): List[ColumnDetails] = {
columns
.zip(orders)
.map{case(c,order) => new ColumnDetails(c.columnMeta.getName,
new ClusteringColumn(OrderConverter.convert(order)),
c.columnMeta.getType, extractIndexDetail(c))}
}
private def convertPartitionKeys(columns: List[ColumnMetaWrapper]): List[ColumnDetails] = {
columns
.map(c => new ColumnDetails(c.columnMeta.getName, PartitionKey, c.columnMeta.getType, extractIndexDetail(c)))
}
}

108
cassandra/src/main/scala/org/apache/zeppelin/cassandra/EnhancedSession.scala Normal file
View file

@ -0,0 +1,108 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import com.datastax.driver.core._
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
import org.apache.zeppelin.interpreter.InterpreterException
/**
* Enhance the Java driver session
* with special statements
* to describe schema
*/
class EnhancedSession(val session: Session) {
val clusterDisplay = DisplaySystem.ClusterDisplay
val keyspaceDisplay = DisplaySystem.KeyspaceDisplay
val tableDisplay = DisplaySystem.TableDisplay
val udtDisplay = DisplaySystem.UDTDisplay
val helpDisplay = DisplaySystem.HelpDisplay
private val noResultDisplay = DisplaySystem.NoResultDisplay
val HTML_MAGIC = "%html \n"
val displayNoResult: String = HTML_MAGIC + noResultDisplay.formatNoResult
def displayExecutionStatistics(query: String, execInfo: ExecutionInfo): String = {
HTML_MAGIC + noResultDisplay.noResultWithExecutionInfo(query, execInfo)
}
private def execute(describeCluster: DescribeClusterCmd): String = {
val metaData = session.getCluster.getMetadata
HTML_MAGIC + clusterDisplay.formatClusterOnly(describeCluster.statement, metaData)
}
private def execute(describeKeyspaces: DescribeKeyspacesCmd): String = {
val metaData = session.getCluster.getMetadata
HTML_MAGIC + clusterDisplay.formatClusterContent(describeKeyspaces.statement, metaData)
}
private def execute(describeTables: DescribeTablesCmd): String = {
val metadata: Metadata = session.getCluster.getMetadata
HTML_MAGIC + clusterDisplay.formatAllTables(describeTables.statement,metadata)
}
private def execute(describeKeyspace: DescribeKeyspaceCmd): String = {
val keyspace: String = describeKeyspace.keyspace
val metadata: KeyspaceMetadata = session.getCluster.getMetadata.getKeyspace(keyspace)
HTML_MAGIC + keyspaceDisplay.formatKeyspaceContent(describeKeyspace.statement, metadata)
}
private def execute(describeTable: DescribeTableCmd): String = {
val metaData = session.getCluster.getMetadata
val tableName: String = describeTable.table
val keyspace: String = describeTable.keyspace.orElse(Option(session.getLoggedKeyspace)).getOrElse("system")
Option(metaData.getKeyspace(keyspace)).flatMap(ks => Option(ks.getTable(tableName))) match {
case Some(tableMeta) => HTML_MAGIC + tableDisplay.format(describeTable.statement, tableMeta, true)
case None => throw new InterpreterException(s"Cannot find table $keyspace.$tableName")
}
}
private def execute(describeUDT: DescribeUDTCmd): String = {
val metaData = session.getCluster.getMetadata
val keyspace: String = describeUDT.keyspace.orElse(Option(session.getLoggedKeyspace)).getOrElse("system")
val udtName: String = describeUDT.udtName
Option(metaData.getKeyspace(keyspace)).flatMap(ks => Option(ks.getUserType(udtName))) match {
case Some(userType) => HTML_MAGIC + udtDisplay.format(describeUDT.statement, userType, true)
case None => throw new InterpreterException(s"Cannot find type $keyspace.$udtName")
}
}
private def execute(helpCmd: HelpCmd): String = {
HTML_MAGIC + helpDisplay.formatHelp()
}
def execute(st: Any): Any = {
st match {
case x:DescribeClusterCmd => execute(x)
case x:DescribeKeyspacesCmd => execute(x)
case x:DescribeTablesCmd => execute(x)
case x:DescribeKeyspaceCmd => execute(x)
case x:DescribeTableCmd => execute(x)
case x:DescribeUDTCmd => execute(x)
case x:HelpCmd => execute(x)
case x:Statement => session.execute(x)
case _ => throw new InterpreterException(s"Cannot execute statement '$st' of type ${st.getClass}")
}
}
}

411
cassandra/src/main/scala/org/apache/zeppelin/cassandra/InterpreterLogic.scala Normal file
View file

@ -0,0 +1,411 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import java.io.{ByteArrayOutputStream, PrintStream}
import java.net.InetAddress
import java.nio.ByteBuffer
import java.text.SimpleDateFormat
import java.util
import java.util.Date
import java.util.concurrent.ConcurrentHashMap
import com.datastax.driver.core.DataType.Name._
import com.datastax.driver.core._
import com.datastax.driver.core.exceptions.DriverException
import com.datastax.driver.core.policies.{LoggingRetryPolicy, FallthroughRetryPolicy, DowngradingConsistencyRetryPolicy, Policies}
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
import org.apache.zeppelin.display.Input.ParamOption
import org.apache.zeppelin.interpreter.InterpreterResult.Code
import org.apache.zeppelin.interpreter.{InterpreterException, InterpreterResult, InterpreterContext}
import org.slf4j.LoggerFactory
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
/**
* Value object to store runtime query parameters
* @param consistency consistency level
* @param serialConsistency serial consistency level
* @param timestamp timestamp
* @param retryPolicy retry policy
* @param fetchSize query fetch size
*/
case class CassandraQueryOptions(consistency: Option[ConsistencyLevel],
serialConsistency:Option[ConsistencyLevel],
timestamp: Option[Long],
retryPolicy: Option[RetryPolicy],
fetchSize: Option[Int])
/**
* Singleton object to store constants
*/
object InterpreterLogic {
val CHOICES_SEPARATOR : String = """\|"""
val VARIABLE_PATTERN = """\{\{[^}]+\}\}""".r
val SIMPLE_VARIABLE_DEFINITION_PATTERN = """\{\{([^=]+)=([^=]+)\}\}""".r
val MULTIPLE_CHOICES_VARIABLE_DEFINITION_PATTERN = """\{\{([^=]+)=((?:[^=]+\|)+[^|]+)\}\}""".r
val STANDARD_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"
val ACCURATE_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"
val defaultRetryPolicy = Policies.defaultRetryPolicy()
val downgradingConsistencyRetryPolicy = DowngradingConsistencyRetryPolicy.INSTANCE
val fallThroughRetryPolicy = FallthroughRetryPolicy.INSTANCE
val loggingDefaultRetryPolicy = new LoggingRetryPolicy(defaultRetryPolicy)
val loggingDownGradingRetryPolicy = new LoggingRetryPolicy(downgradingConsistencyRetryPolicy)
val loggingFallThrougRetryPolicy = new LoggingRetryPolicy(fallThroughRetryPolicy)
val preparedStatements : mutable.Map[String,PreparedStatement] = new ConcurrentHashMap[String,PreparedStatement]().asScala
val logger = LoggerFactory.getLogger(classOf[InterpreterLogic])
val paragraphParser = new ParagraphParser
val boundValuesParser = new BoundValuesParser
}
/**
* Real class to implement the
* interpreting logic of CQL statements
* and parameters blocks
*
* @param session java driver session
*/
class InterpreterLogic(val session: Session) {
val enhancedSession: EnhancedSession = new EnhancedSession(session)
import InterpreterLogic._
def interpret(session:Session, stringStatements : String, context: InterpreterContext): InterpreterResult = {
logger.info(s"Executing CQL statements : \n\n$stringStatements\n")
try {
val protocolVersion = session.getCluster.getConfiguration.getProtocolOptions.getProtocolVersionEnum
val queries:List[AnyBlock] = parseInput(stringStatements)
val queryOptions = extractQueryOptions(queries
.filter(_.blockType == ParameterBlock)
.map(_.get[QueryParameters]))
logger.info(s"Current Cassandra query options = $queryOptions")
val queryStatements = queries.filter(_.blockType == StatementBlock).map(_.get[QueryStatement])
//Remove prepared statements
queryStatements
.filter(_.statementType == RemovePrepareStatementType)
.map(_.getStatement[RemovePrepareStm])
.foreach(remove => {
logger.debug(s"Removing prepared statement '${remove.name}'")
preparedStatements.remove(remove.name)
})
//Update prepared statement maps
queryStatements
.filter(_.statementType == PrepareStatementType)
.map(_.getStatement[PrepareStm])
.foreach(statement => {
logger.debug(s"Get or prepare statement '${statement.name}' : ${statement.query}")
preparedStatements.getOrElseUpdate(statement.name,session.prepare(statement.query))
})
val statements: List[Any] = queryStatements
.filter(st => (st.statementType != PrepareStatementType) && (st.statementType != RemovePrepareStatementType))
.map{
case x:SimpleStm => generateSimpleStatement(x, queryOptions, context)
case x:BatchStm => {
val builtStatements: List[Statement] = x.statements.map {
case st:SimpleStm => generateSimpleStatement(st, queryOptions, context)
case st:BoundStm => generateBoundStatement(st, queryOptions, context)
case _ => throw new InterpreterException(s"Unknown statement type")
}
generateBatchStatement(x.batchType, queryOptions, builtStatements)
}
case x:BoundStm => generateBoundStatement(x, queryOptions, context)
case x:DescribeCommandStatement => x
case x:HelpCmd => x
case x => throw new InterpreterException(s"Unknown statement type : ${x}")
}
val results: List[(Any,Any)] = for (statement <- statements) yield (enhancedSession.execute(statement),statement)
if (results.nonEmpty) {
results.last match {
case(res: ResultSet, st: Statement) => buildResponseMessage((res, st), protocolVersion)
case(output: String, _) => new InterpreterResult(Code.SUCCESS, output)
case _ => throw new InterpreterException(s"Cannot parse result type : ${results.last}")
}
} else {
new InterpreterResult(Code.SUCCESS, enhancedSession.displayNoResult)
}
} catch {
case dex: DriverException => {
logger.error(dex.getMessage, dex)
new InterpreterResult(Code.ERROR, parseException(dex))
}
case pex:ParsingException => {
logger.error(pex.getMessage, pex)
new InterpreterResult(Code.ERROR, pex.getMessage)
}
case iex: InterpreterException => {
logger.error(iex.getMessage, iex)
new InterpreterResult(Code.ERROR, iex.getMessage)
}
case ex: java.lang.Exception => {
logger.error(ex.getMessage, ex)
new InterpreterResult(Code.ERROR, parseException(ex))
}
}
}
def buildResponseMessage(lastResultSet: (ResultSet,Statement), protocolVersion: ProtocolVersion): InterpreterResult = {
val output = new StringBuilder()
val rows: collection.mutable.ArrayBuffer[Row] = ArrayBuffer()
val iterator: util.Iterator[Row] = lastResultSet._1.iterator()
while (iterator.hasNext) {
rows.append(iterator.next())
}
val columnsDefinitions: List[(String, DataType)] = lastResultSet._1
.getColumnDefinitions
.asList
.toList // Java list -> Scala list
.map(definition => (definition.getName, definition.getType))
if (rows.nonEmpty) {
// Create table headers
output
.append("%table ")
.append(columnsDefinitions.map { case (columnName, _) => columnName }.mkString("\t")).append("\n")
// Deserialize Data
rows.foreach {
row => {
val data = columnsDefinitions.map {
case (name, dataType) => {
if (row.isNull(name)) null else dataType.deserialize(row.getBytesUnsafe(name), protocolVersion)
}
}
output.append(data.mkString("\t")).append("\n")
}
}
} else {
val lastQuery: String = lastResultSet._2.toString
val executionInfo: ExecutionInfo = lastResultSet._1.getExecutionInfo
output.append(enhancedSession.displayExecutionStatistics(lastQuery, executionInfo))
}
val result: String = output.toString()
logger.debug(s"CQL result : \n\n$result\n")
new InterpreterResult(Code.SUCCESS, result)
}
def parseInput(input:String): List[AnyBlock] = {
val parsingResult: ParagraphParser#ParseResult[List[AnyBlock]] = paragraphParser.parseAll(paragraphParser.queries, input)
parsingResult match {
case paragraphParser.Success(blocks,_) => blocks
case paragraphParser.Failure(msg,next) => {
throw new InterpreterException(s"Error parsing input:\n\t'$input'\nDid you forget to add ; (semi-colon) at the end of each CQL statement ?")
}
case paragraphParser.Error(msg,next) => {
throw new InterpreterException(s"Error parsing input:\n\t'$input'\nDid you forget to add ; (semi-colon) at the end of each CQL statement ?")
}
case _ => throw new InterpreterException(s"Error parsing input: $input")
}
}
def extractQueryOptions(parameters: List[QueryParameters]): CassandraQueryOptions = {
logger.debug(s"Extracting query options from $parameters")
val consistency: Option[ConsistencyLevel] = parameters
.filter(_.paramType == ConsistencyParam)
.map(_.getParam[Consistency])
.flatMap(x => Option(x.value))
.headOption
val serialConsistency: Option[ConsistencyLevel] = parameters
.filter(_.paramType == SerialConsistencyParam)
.map(_.getParam[SerialConsistency])
.flatMap(x => Option(x.value))
.headOption
val timestamp: Option[Long] = parameters
.filter(_.paramType == TimestampParam)
.map(_.getParam[Timestamp])
.flatMap(x => Option(x.value))
.headOption
val retryPolicy: Option[RetryPolicy] = parameters
.filter(_.paramType == RetryPolicyParam)
.map(_.getParam[RetryPolicy])
.headOption
val fetchSize: Option[Int] = parameters
.filter(_.paramType == FetchSizeParam)
.map(_.getParam[FetchSize])
.flatMap(x => Option(x.value))
.headOption
CassandraQueryOptions(consistency,serialConsistency, timestamp, retryPolicy, fetchSize)
}
def generateSimpleStatement(st: SimpleStm, options: CassandraQueryOptions,context: InterpreterContext): SimpleStatement = {
logger.debug(s"Generating simple statement : '${st.text}'")
val statement = new SimpleStatement(maybeExtractVariables(st.text, context))
applyQueryOptions(options, statement)
statement
}
def generateBoundStatement(st: BoundStm, options: CassandraQueryOptions,context: InterpreterContext): BoundStatement = {
logger.debug(s"Generating bound statement with name : '${st.name}' and bound values : ${st.values}")
preparedStatements.get(st.name) match {
case Some(ps) => {
val boundValues = maybeExtractVariables(st.values, context)
createBoundStatement(st.name, ps, boundValues)
}
case None => throw new InterpreterException(s"The statement '${st.name}' can not be bound to values. " +
s"Are you sure you did prepare it with @prepare[${st.name}] ?")
}
}
def generateBatchStatement(batchType: BatchStatement.Type, options: CassandraQueryOptions, statements: List[Statement]): BatchStatement = {
logger.debug(s"""Generating batch statement of type '${batchType} for ${statements.mkString(",")}'""")
val batch = new BatchStatement(batchType)
statements.foreach(batch.add(_))
applyQueryOptions(options, batch)
batch
}
def maybeExtractVariables(statement: String, context: InterpreterContext): String = {
def extractVariableAndDefaultValue(statement: String, exp: String):String = {
exp match {
case MULTIPLE_CHOICES_VARIABLE_DEFINITION_PATTERN(variable,choices) => {
val escapedExp: String = exp.replaceAll( """\{""", """\\{""").replaceAll( """\}""", """\\}""").replaceAll("""\|""","""\\|""")
val listChoices:List[String] = choices.trim.split(CHOICES_SEPARATOR).toList
val paramOptions= listChoices.map(choice => new ParamOption(choice, choice))
val selected = context.getGui.select(variable, listChoices.head, paramOptions.toArray)
statement.replaceAll(escapedExp,selected.toString)
}
case SIMPLE_VARIABLE_DEFINITION_PATTERN(variable,defaultVal) => {
val escapedExp: String = exp.replaceAll( """\{""", """\\{""").replaceAll( """\}""", """\\}""")
val value = context.getGui.input(variable,defaultVal)
statement.replaceAll(escapedExp,value.toString)
}
case _ => throw new ParsingException(s"Invalid bound variable definition for '$exp' in '$statement'. It should be of form 'variable=defaultValue' or 'variable=value1|value2|...|valueN'")
}
}
VARIABLE_PATTERN.findAllIn(statement).foldLeft(statement)(extractVariableAndDefaultValue _)
}
def applyQueryOptions(options: CassandraQueryOptions, statement: Statement): Unit = {
options.consistency.foreach(statement.setConsistencyLevel(_))
options.serialConsistency.foreach(statement.setSerialConsistencyLevel(_))
options.timestamp.foreach(statement.setDefaultTimestamp(_))
options.retryPolicy.foreach {
case DefaultRetryPolicy => statement.setRetryPolicy(defaultRetryPolicy)
case DowngradingRetryPolicy => statement.setRetryPolicy(downgradingConsistencyRetryPolicy)
case FallThroughRetryPolicy => statement.setRetryPolicy(fallThroughRetryPolicy)
case LoggingDefaultRetryPolicy => statement.setRetryPolicy(loggingDefaultRetryPolicy)
case LoggingDowngradingRetryPolicy => statement.setRetryPolicy(loggingDownGradingRetryPolicy)
case LoggingFallThroughRetryPolicy => statement.setRetryPolicy(loggingFallThrougRetryPolicy)
case _ => throw new InterpreterException(s"""Unknown retry policy ${options.retryPolicy.getOrElse("???")}""")
}
options.fetchSize.foreach(statement.setFetchSize(_))
}
private def createBoundStatement(name: String, ps: PreparedStatement, rawBoundValues: String): BoundStatement = {
val dataTypes = ps.getVariables.toList
.map(cfDef => cfDef.getType)
val boundValuesAsText = parseBoundValues(name,rawBoundValues)
if(dataTypes.size != boundValuesAsText.size) throw new InterpreterException(s"Invalid @bind values for prepared statement '$name'. " +
s"Prepared parameters has ${dataTypes.size} variables whereas bound values have ${boundValuesAsText.size} parameters ...")
val convertedValues: List[AnyRef] = boundValuesAsText
.zip(dataTypes).map {
case (value, dataType) => {
if(value.trim == "null") {
null
} else {
dataType.getName match {
case (ASCII | TEXT | VARCHAR) => value.trim.replaceAll("(?<!')'","")
case (INT | VARINT) => value.trim.toInt
case (BIGINT | COUNTER) => value.trim.toLong
case BLOB => ByteBuffer.wrap(value.trim.getBytes)
case BOOLEAN => value.trim.toBoolean
case DECIMAL => BigDecimal(value.trim)
case DOUBLE => value.trim.toDouble
case FLOAT => value.trim.toFloat
case INET => InetAddress.getByName(value.trim)
case TIMESTAMP => parseDate(value.trim)
case (UUID | TIMEUUID) => java.util.UUID.fromString(value.trim)
case LIST => dataType.parse(boundValuesParser.parse(boundValuesParser.list, value).get)
case SET => dataType.parse(boundValuesParser.parse(boundValuesParser.set, value).get)
case MAP => dataType.parse(boundValuesParser.parse(boundValuesParser.map, value).get)
case UDT => dataType.parse(boundValuesParser.parse(boundValuesParser.udt, value).get)
case TUPLE => dataType.parse(boundValuesParser.parse(boundValuesParser.tuple, value).get)
case _ => throw new InterpreterException(s"Cannot parse data of type : ${dataType.toString}")
}
}
}
}.asInstanceOf[List[AnyRef]]
ps.bind(convertedValues.toArray: _*)
}
protected def parseBoundValues(psName: String, boundValues: String): List[String] = {
val result: BoundValuesParser#ParseResult[List[String]] = boundValuesParser.parseAll(boundValuesParser.values, boundValues)
result match {
case boundValuesParser.Success(list,_) => list
case _ => throw new InterpreterException(s"Cannot parse bound values for prepared statement '$psName' : $boundValues. Did you forget to wrap text with ' (simple quote) ?")
}
}
def parseDate(dateString: String): Date = {
dateString match {
case boundValuesParser.STANDARD_DATE_PATTERN(datePattern) => new SimpleDateFormat(STANDARD_DATE_FORMAT).parse(datePattern)
case boundValuesParser.ACCURATE_DATE_PATTERN(datePattern) => new SimpleDateFormat(ACCURATE_DATE_FORMAT).parse(datePattern)
case _ => throw new InterpreterException(s"Cannot parse date '$dateString'. " +
s"Accepted formats : $STANDARD_DATE_FORMAT OR $ACCURATE_DATE_FORMAT");
}
}
def parseException(ex: Exception): String = {
val os = new ByteArrayOutputStream()
val ps = new PrintStream(os)
ex.printStackTrace(ps)
os.toString("UTF-8")
}
}

430
cassandra/src/main/scala/org/apache/zeppelin/cassandra/JavaDriverConfig.scala Normal file
View file

@ -0,0 +1,430 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import java.lang.Boolean._
import com.datastax.driver.core.HostDistance._
import com.datastax.driver.core.ProtocolOptions.Compression
import com.datastax.driver.core._
import com.datastax.driver.core.policies._
import org.apache.commons.lang3.StringUtils._
import org.apache.zeppelin.interpreter.Interpreter
import org.apache.zeppelin.cassandra.CassandraInterpreter._
import org.slf4j.LoggerFactory
/**
* Utility class to extract and configure the Java driver
*/
class JavaDriverConfig {
val LOGGER = LoggerFactory.getLogger(classOf[JavaDriverConfig])
def getSocketOptions(intpr: Interpreter): SocketOptions = {
val socketOptions: SocketOptions = new SocketOptions
val socketOptionsInfo: StringBuilder = new StringBuilder("Socket options : \n\n")
val connectTimeoutMillis: Int = intpr.getProperty(CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS).toInt
socketOptions.setConnectTimeoutMillis(connectTimeoutMillis)
socketOptionsInfo
.append("\t")
.append(CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS)
.append(" : ")
.append(connectTimeoutMillis).append("\n")
val readTimeoutMillis: Int = intpr.getProperty(CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS).toInt
socketOptions.setReadTimeoutMillis(readTimeoutMillis)
socketOptionsInfo
.append("\t")
.append(CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS)
.append(" : ")
.append(readTimeoutMillis).append("\n")
val tcpNoDelay: Boolean = parseBoolean(intpr.getProperty(CASSANDRA_SOCKET_TCP_NO_DELAY))
socketOptions.setTcpNoDelay(tcpNoDelay)
socketOptionsInfo
.append("\t")
.append(CASSANDRA_SOCKET_TCP_NO_DELAY)
.append(" : ")
.append(tcpNoDelay)
.append("\n")
val keepAlive: String = intpr.getProperty(CASSANDRA_SOCKET_KEEP_ALIVE)
if (isNotBlank(keepAlive)) {
val keepAliveValue: Boolean = parseBoolean(keepAlive)
socketOptions.setKeepAlive(keepAliveValue)
socketOptionsInfo
.append("\t")
.append(CASSANDRA_SOCKET_KEEP_ALIVE)
.append(" : ")
.append(keepAliveValue).append("\n")
}
val receivedBuffSize: String = intpr.getProperty(CASSANDRA_SOCKET_RECEIVED_BUFFER_SIZE_BYTES)
if (isNotBlank(receivedBuffSize)) {
val receiveBufferSizeValue: Int = receivedBuffSize.toInt
socketOptions.setReceiveBufferSize(receiveBufferSizeValue)
socketOptionsInfo
.append("\t")
.append(CASSANDRA_SOCKET_RECEIVED_BUFFER_SIZE_BYTES)
.append(" : ")
.append(receiveBufferSizeValue)
.append("\n")
}
val sendBuffSize: String = intpr.getProperty(CASSANDRA_SOCKET_SEND_BUFFER_SIZE_BYTES)
if (isNotBlank(sendBuffSize)) {
val sendBufferSizeValue: Int = sendBuffSize.toInt
socketOptions.setSendBufferSize(sendBufferSizeValue)
socketOptionsInfo
.append("\t")
.append(CASSANDRA_SOCKET_SEND_BUFFER_SIZE_BYTES)
.append(" : ")
.append(sendBufferSizeValue)
.append("\n")
}
val reuseAddress: String = intpr.getProperty(CASSANDRA_SOCKET_REUSE_ADDRESS)
if (isNotBlank(reuseAddress)) {
val reuseAddressValue: Boolean = parseBoolean(reuseAddress)
socketOptions.setReuseAddress(reuseAddressValue)
socketOptionsInfo
.append("\t")
.append(CASSANDRA_SOCKET_REUSE_ADDRESS)
.append(" : ")
.append(reuseAddressValue)
.append("\n")
}
val soLinger: String = intpr.getProperty(CASSANDRA_SOCKET_SO_LINGER)
if (isNotBlank(soLinger)) {
val soLingerValue: Int = soLinger.toInt
socketOptions.setSoLinger(soLingerValue)
socketOptionsInfo
.append("\t")
.append(CASSANDRA_SOCKET_SO_LINGER)
.append(" : ")
.append(soLingerValue)
.append("\n")
}
LOGGER.debug(socketOptionsInfo.append("\n").toString)
return socketOptions
}
def getQueryOptions(intpr: Interpreter): QueryOptions = {
val queryOptions: QueryOptions = new QueryOptions
val queryOptionsInfo: StringBuilder = new StringBuilder("Query options : \n\n")
val consistencyLevel: ConsistencyLevel = ConsistencyLevel.valueOf(intpr.getProperty(CASSANDRA_QUERY_DEFAULT_CONSISTENCY))
queryOptions.setConsistencyLevel(consistencyLevel)
queryOptionsInfo
.append("\t")
.append(CASSANDRA_QUERY_DEFAULT_CONSISTENCY)
.append(" : ")
.append(consistencyLevel)
.append("\n")
val serialConsistencyLevel: ConsistencyLevel = ConsistencyLevel.valueOf(intpr.getProperty(CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY))
queryOptions.setSerialConsistencyLevel(serialConsistencyLevel)
queryOptionsInfo
.append("\t")
.append(CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY)
.append(" : ")
.append(serialConsistencyLevel)
.append("\n")
val fetchSize: Int = intpr.getProperty(CASSANDRA_QUERY_DEFAULT_FETCH_SIZE).toInt
queryOptions.setFetchSize(fetchSize)
queryOptionsInfo
.append("\t")
.append(CASSANDRA_QUERY_DEFAULT_FETCH_SIZE)
.append(" : ")
.append(fetchSize)
.append("\n")
val defaultIdempotence: Boolean = parseBoolean(intpr.getProperty(CASSANDRA_QUERY_DEFAULT_IDEMPOTENCE))
queryOptions.setDefaultIdempotence(defaultIdempotence)
queryOptionsInfo
.append("\t")
.append(CASSANDRA_QUERY_DEFAULT_IDEMPOTENCE)
.append(" : ")
.append(defaultIdempotence)
.append("\n")
LOGGER.debug(queryOptionsInfo.append("\n").toString)
return queryOptions
}
def getProtocolVersion(intpr: Interpreter): ProtocolVersion = {
val protocolVersion: String = intpr.getProperty(CASSANDRA_PROTOCOL_VERSION)
LOGGER.debug("Protocol version : " + protocolVersion)
protocolVersion match {
case "1" =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "1"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "128"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "128"
return ProtocolVersion.V1
case "2" =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "8"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "2"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "2"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "100"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "1"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "128"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "128"
return ProtocolVersion.V2
case "3" =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "800"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "200"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256"
return ProtocolVersion.V3
case _ =>
DEFAULT_MAX_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_MAX_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_LOCAL = "1"
DEFAULT_CORE_CONNECTION_PER_HOST_REMOTE = "1"
DEFAULT_NEW_CONNECTION_THRESHOLD_LOCAL = "800"
DEFAULT_NEW_CONNECTION_THRESHOLD_REMOTE = "200"
DEFAULT_MAX_REQUEST_PER_CONNECTION_LOCAL = "1024"
DEFAULT_MAX_REQUEST_PER_CONNECTION_REMOTE = "256"
return ProtocolVersion.NEWEST_SUPPORTED
}
}
def getPoolingOptions(intpr: Interpreter): PoolingOptions = {
val poolingOptions: PoolingOptions = new PoolingOptions
val poolingOptionsInfo: StringBuilder = new StringBuilder("Pooling options : \n\n")
val maxConnPerHostLocal: Int = intpr.getProperty(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL).toInt
poolingOptions.setMaxConnectionsPerHost(LOCAL, maxConnPerHostLocal)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL)
.append(" : ")
.append(maxConnPerHostLocal)
.append("\n")
val maxConnPerHostRemote: Int = intpr.getProperty(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE).toInt
poolingOptions.setMaxConnectionsPerHost(REMOTE, maxConnPerHostRemote)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE)
.append(" : ")
.append(maxConnPerHostRemote)
.append("\n")
val coreConnPerHostLocal: Int = intpr.getProperty(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL).toInt
poolingOptions.setCoreConnectionsPerHost(LOCAL, coreConnPerHostLocal)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL)
.append(" : ")
.append(coreConnPerHostLocal)
.append("\n")
val coreConnPerHostRemote: Int = intpr.getProperty(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE).toInt
poolingOptions.setCoreConnectionsPerHost(REMOTE, coreConnPerHostRemote)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE)
.append(" : ")
.append(coreConnPerHostRemote)
.append("\n")
val newConnThresholdLocal: Int = intpr.getProperty(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL).toInt
poolingOptions.setNewConnectionThreshold(LOCAL, newConnThresholdLocal)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL)
.append(" : ")
.append(newConnThresholdLocal)
.append("\n")
val newConnThresholdRemote: Int = intpr.getProperty(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE).toInt
poolingOptions.setNewConnectionThreshold(REMOTE, newConnThresholdRemote)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE)
.append(" : ")
.append(newConnThresholdRemote)
.append("\n")
val maxReqPerConnLocal: Int = intpr.getProperty(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL).toInt
poolingOptions.setMaxRequestsPerConnection(LOCAL, maxReqPerConnLocal)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL)
.append(" : ")
.append(maxReqPerConnLocal)
.append("\n")
val maxReqPerConnRemote: Int = intpr.getProperty(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE).toInt
poolingOptions.setMaxRequestsPerConnection(REMOTE, maxReqPerConnRemote)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE)
.append(" : ")
.append(maxReqPerConnRemote)
.append("\n")
val heartbeatIntervalSeconds: Int = intpr.getProperty(CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS).toInt
poolingOptions.setHeartbeatIntervalSeconds(heartbeatIntervalSeconds)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS)
.append(" : ")
.append(heartbeatIntervalSeconds)
.append("\n")
val idleTimeoutSeconds: Int = intpr.getProperty(CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS).toInt
poolingOptions.setIdleTimeoutSeconds(idleTimeoutSeconds)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS)
.append(" : ")
.append(idleTimeoutSeconds)
.append("\n")
val poolTimeoutMillis: Int = intpr.getProperty(CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS).toInt
poolingOptions.setPoolTimeoutMillis(poolTimeoutMillis)
poolingOptionsInfo
.append("\t")
.append(CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS)
.append(" : ")
.append(poolTimeoutMillis)
.append("\n")
LOGGER.debug(poolingOptionsInfo.append("\n").toString)
return poolingOptions
}
def getCompressionProtocol(intpr: Interpreter): ProtocolOptions.Compression = {
var compression: ProtocolOptions.Compression = null
val compressionProtocol: String = intpr.getProperty(CASSANDRA_COMPRESSION_PROTOCOL)
LOGGER.debug("Compression protocol : " + compressionProtocol)
if (compressionProtocol == null) "NONE"
else compressionProtocol.toUpperCase match {
case "NONE" =>
compression = Compression.NONE
case "SNAPPY" =>
compression = Compression.SNAPPY
case "LZ4" =>
compression = Compression.LZ4
case _ =>
compression = Compression.NONE
}
return compression
}
def getLoadBalancingPolicy(intpr: Interpreter): LoadBalancingPolicy = {
val loadBalancingPolicy: String = intpr.getProperty(CASSANDRA_LOAD_BALANCING_POLICY)
LOGGER.debug("Load Balancing Policy : " + loadBalancingPolicy)
if (isBlank(loadBalancingPolicy) || (DEFAULT_POLICY == loadBalancingPolicy)) {
return Policies.defaultLoadBalancingPolicy
}
else {
try {
return (Class.forName(loadBalancingPolicy).asInstanceOf[Class[LoadBalancingPolicy]]).newInstance
}
catch {
case e: Any => {
e.printStackTrace
throw new RuntimeException("Cannot instantiate " + CASSANDRA_LOAD_BALANCING_POLICY + " = " + loadBalancingPolicy)
}
}
}
}
def getRetryPolicy(intpr: Interpreter): RetryPolicy = {
val retryPolicy: String = intpr.getProperty(CASSANDRA_RETRY_POLICY)
LOGGER.debug("Retry Policy : " + retryPolicy)
if (isBlank(retryPolicy) || (DEFAULT_POLICY == retryPolicy)) {
return Policies.defaultRetryPolicy
}
else {
try {
return (Class.forName(retryPolicy).asInstanceOf[Class[RetryPolicy]]).newInstance
}
catch {
case e: Any => {
e.printStackTrace
throw new RuntimeException("Cannot instantiate " + CASSANDRA_RETRY_POLICY + " = " + retryPolicy)
}
}
}
}
def getReconnectionPolicy(intpr: Interpreter): ReconnectionPolicy = {
val reconnectionPolicy: String = intpr.getProperty(CASSANDRA_RECONNECTION_POLICY)
LOGGER.debug("Reconnection Policy : " + reconnectionPolicy)
if (isBlank(reconnectionPolicy) || (DEFAULT_POLICY == reconnectionPolicy)) {
return Policies.defaultReconnectionPolicy
}
else {
try {
return (Class.forName(reconnectionPolicy).asInstanceOf[Class[ReconnectionPolicy]]).newInstance
}
catch {
case e: Any => {
e.printStackTrace
throw new RuntimeException("Cannot instantiate " + CASSANDRA_RECONNECTION_POLICY + " = " + reconnectionPolicy)
}
}
}
}
def getSpeculativeExecutionPolicy(intpr: Interpreter): SpeculativeExecutionPolicy = {
val specExecPolicy: String = intpr.getProperty(CASSANDRA_SPECULATIVE_EXECUTION_POLICY)
LOGGER.debug("Speculative Execution Policy : " + specExecPolicy)
if (isBlank(specExecPolicy) || (DEFAULT_POLICY == specExecPolicy)) {
return Policies.defaultSpeculativeExecutionPolicy
}
else {
try {
return (Class.forName(specExecPolicy).asInstanceOf[Class[SpeculativeExecutionPolicy]]).newInstance
}
catch {
case e: Any => {
e.printStackTrace
throw new RuntimeException("Cannot instantiate " + CASSANDRA_SPECULATIVE_EXECUTION_POLICY + " = " + specExecPolicy)
}
}
}
}
}

67
cassandra/src/main/scala/org/apache/zeppelin/cassandra/MetaDataHierarchy.scala Normal file
View file

@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import java.util.UUID
import com.datastax.driver.core.utils.UUIDs
import com.datastax.driver.core.{DataType, TableMetadata}
import scala.util.parsing.json.JSONObject
/**
* Define a hierarchy for CQL meta data
*/
object MetaDataHierarchy {
object OrderConverter {
def convert(clusteringOrder: TableMetadata.Order): ClusteringOrder = {
clusteringOrder match {
case TableMetadata.Order.ASC => ASC
case TableMetadata.Order.DESC => DESC
}
}
}
sealed trait ClusteringOrder
object ASC extends ClusteringOrder
object DESC extends ClusteringOrder
sealed trait ColumnType
object PartitionKey extends ColumnType
case class ClusteringColumn(order: ClusteringOrder) extends ColumnType
object StaticColumn extends ColumnType
object NormalColumn extends ColumnType
case class IndexDetails(name: String, info: String)
case class ColumnDetails(name: String, columnType: ColumnType, dataType: DataType, index: Option[IndexDetails])
case class ClusterDetails(name: String, partitioner: String)
case class ClusterContent(clusterName: String, clusterDetails: String, keyspaces: List[(UUID, String, String)])
case class AllTables(tables: Map[String,List[String]])
case class KeyspaceDetails(name: String, replication: Map[String,String], durableWrites: Boolean, asCQL: String, uniqueId: UUID = UUIDs.timeBased()) {
def getReplicationMap: String = {
JSONObject(replication).toString().replaceAll(""""""","'")
}
}
case class KeyspaceContent(keyspaceName: String, keyspaceDetails: String, tables: List[(UUID,String, String)], udts: List[(UUID, String, String)])
case class TableDetails(tableName: String, columns: List[ColumnDetails], asCQL: String, uniqueId: UUID = UUIDs.timeBased())
case class UDTDetails(typeName: String, columns: List[ColumnDetails], asCQL: String, uniqueId: UUID = UUIDs.timeBased())
}

267
cassandra/src/main/scala/org/apache/zeppelin/cassandra/ParagraphParser.scala Normal file
View file

@ -0,0 +1,267 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import com.datastax.driver.core._
import org.apache.zeppelin.cassandra.CassandraInterpreter._
import org.apache.zeppelin.interpreter.InterpreterException
import scala.util.parsing.combinator._
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
object ParagraphParser {
val CONSISTENCY_LEVEL_PATTERN = ConsistencyLevel.values().toList
.map(_.name()).filter(!_.contains("SERIAL")).mkString("""^\s*@consistency\s*=\s*(""", "|" , """)\s*$""").r
val SERIAL_CONSISTENCY_LEVEL_PATTERN = ConsistencyLevel.values().toList
.map(_.name()).filter(_.contains("SERIAL")).mkString("""^\s*@serialConsistency\s*=\s*(""", "|", """)\s*$""").r
val TIMESTAMP_PATTERN = """^\s*@timestamp\s*=\s*([0-9]+)\s*$""".r
val RETRY_POLICIES_PATTERN = List(DEFAULT_POLICY,DOWNGRADING_CONSISTENCY_RETRY, FALLTHROUGH_RETRY,
LOGGING_DEFAULT_RETRY, LOGGING_DOWNGRADING_RETRY, LOGGING_FALLTHROUGH_RETRY)
.mkString("""^\s*@retryPolicy\s*=\s*(""", "|" , """)\s*$""").r
val FETCHSIZE_PATTERN = """^\s*@fetchSize\s*=\s*([0-9]+)\s*$""".r
val SIMPLE_STATEMENT_PATTERN = """([^;]+;)""".r
val PREPARE_STATEMENT_PATTERN = """^\s*@prepare\[([^]]+)\]\s*=\s*([^;]+)$""".r
val REMOVE_PREPARE_STATEMENT_PATTERN = """^\s*@remove_prepare\[([^]]+)\]\s*$""".r
val BIND_PATTERN = """^\s*@bind\[([^]]+)\](?:=([^;]+))?""".r
val BATCH_PATTERN = """^(?i)\s*BEGIN\s+(UNLOGGED|COUNTER)?\s*BATCH""".r
val GENERIC_STATEMENT_PREFIX =
"""(?is)\s*(?:INSERT|UPDATE|DELETE|SELECT|CREATE|UPDATE|
|DROP|GRANT|REVOKE|TRUNCATE|LIST|USE)\s+""".r
val VALID_IDENTIFIER = "[a-z][a-z0-9_]*"
val DESCRIBE_CLUSTER_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+CLUSTER;\s*$""".r
val DESCRIBE_KEYSPACES_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+KEYSPACES;\s*$""".r
val DESCRIBE_TABLES_PATTERN = """^(?i)\s*(?:DESCRIBE|DESC)\s+TABLES;\s*$""".r
val DESCRIBE_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+KEYSPACE\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_TABLE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+TABLE\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+TABLE\s*(""" +
VALID_IDENTIFIER +
""")\.(""" +
VALID_IDENTIFIER +
""");\s*$""").r
val DESCRIBE_TYPE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+TYPE\s*("""+VALID_IDENTIFIER+""");\s*$""").r
val DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN = ("""^(?i)\s*(?:DESCRIBE|DESC)\s+TYPE\s*(""" +
VALID_IDENTIFIER +
""")\.(""" +
VALID_IDENTIFIER +
""");\s*$""").r
val HELP_PATTERN = """^(?i)\s*HELP;\s*$""".r
}
class ParagraphParser extends RegexParsers{
import ParagraphParser._
def singleLineComment: Parser[Comment] = """\s*#.*""".r ^^ {case text => Comment(text.trim.replaceAll("#",""))}
def multiLineComment: Parser[Comment] = """(?s)/\*(.*)\*/""".r ^^ {case text => Comment(text.trim.replaceAll("""/\*""","").replaceAll("""\*/""",""))}
//Query parameters
def consistency: Parser[Consistency] = """\s*@consistency.+""".r ^^ {case x => extractConsistency(x.trim)}
def serialConsistency: Parser[SerialConsistency] = """\s*@serialConsistency.+""".r ^^ {case x => extractSerialConsistency(x.trim)}
def timestamp: Parser[Timestamp] = """\s*@timestamp.+""".r ^^ {case x => extractTimestamp(x.trim)}
def retryPolicy: Parser[RetryPolicy] = """\s*@retryPolicy.+""".r ^^ {case x => extractRetryPolicy(x.trim)}
def fetchSize: Parser[FetchSize] = """\s*@fetchSize.+""".r ^^ {case x => extractFetchSize(x.trim)}
//Statements
def genericStatement: Parser[SimpleStm] = s"""$GENERIC_STATEMENT_PREFIX[^;]+;""".r ^^ {case x => extractSimpleStatement(x.trim)}
def prepare: Parser[PrepareStm] = """\s*@prepare.+""".r ^^ {case x => extractPreparedStatement(x.trim)}
def removePrepare: Parser[RemovePrepareStm] = """\s*@remove_prepare.+""".r ^^ {case x => extractRemovePreparedStatement(x.trim)}
def bind: Parser[BoundStm] = """\s*@bind.+""".r ^^ {case x => extractBoundStatement(x.trim)}
//Meta data
private def describeCluster: Parser[DescribeClusterCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+CLUSTER.*""".r ^^ {extractDescribeClusterCmd(_)}
private def describeKeyspaces: Parser[DescribeKeyspacesCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+KEYSPACES.*""".r ^^ {extractDescribeKeyspacesCmd(_)}
private def describeTables: Parser[DescribeTablesCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+TABLES.*""".r ^^ {extractDescribeTablesCmd(_)}
private def describeKeyspace: Parser[DescribeKeyspaceCmd] = """\s*(?i)(?:DESCRIBE|DESC)\s+KEYSPACE\s+.+""".r ^^ {extractDescribeKeyspaceCmd(_)}
private def describeTable: Parser[DescribeTableCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+TABLE\s+.+""".r ^^ {extractDescribeTableCmd(_)}
private def describeType: Parser[DescribeUDTCmd] = """(?i)\s*(?:DESCRIBE|DESC)\s+TYPE\s+.*""".r ^^ {extractDescribeTypeCmd(_)}
//Help
private def helpCommand: Parser[HelpCmd] = """(?i)\s*HELP.*""".r ^^{extractHelpCmd(_)}
private def beginBatch: Parser[String] = """(?i)\s*BEGIN\s+(UNLOGGED|COUNTER)?\s*BATCH""".r
private def applyBatch: Parser[String] = """(?i)APPLY BATCH;""".r
private def insert: Parser[SimpleStm] = """(?i)INSERT [^;]+;""".r ^^{SimpleStm(_)}
private def update: Parser[SimpleStm] = """(?i)UPDATE [^;]+;""".r ^^{SimpleStm(_)}
private def delete: Parser[SimpleStm] = """(?i)DELETE [^;]+;""".r ^^{SimpleStm(_)}
private def mutationStatements: Parser[List[QueryStatement]] = rep(insert | update | delete | bind)
def batch: Parser[BatchStm] = beginBatch ~ mutationStatements ~ applyBatch ^^ {
case begin ~ cqls ~ end => BatchStm(extractBatchType(begin),cqls)}
def queries:Parser[List[AnyBlock]] = rep(singleLineComment | multiLineComment | consistency | serialConsistency |
timestamp | retryPolicy | fetchSize | removePrepare | prepare | bind | batch | describeCluster | describeKeyspaces |
describeTables | describeKeyspace | describeTable | describeType | helpCommand | genericStatement)
def extractConsistency(text: String): Consistency = {
text match {
case CONSISTENCY_LEVEL_PATTERN(consistency) => Consistency(ConsistencyLevel.valueOf(consistency))
case _ => throw new InterpreterException(s"Invalid syntax for @consistency. " +
s"It should comply to the pattern ${CONSISTENCY_LEVEL_PATTERN.toString}")
}
}
def extractSerialConsistency(text: String): SerialConsistency = {
text match {
case SERIAL_CONSISTENCY_LEVEL_PATTERN(consistency) => SerialConsistency(ConsistencyLevel.valueOf(consistency))
case _ => throw new InterpreterException(s"Invalid syntax for @serialConsistency. " +
s"It should comply to the pattern ${SERIAL_CONSISTENCY_LEVEL_PATTERN.toString}")
}
}
def extractTimestamp(text: String): Timestamp = {
text match {
case TIMESTAMP_PATTERN(timestamp) => Timestamp(timestamp.trim.toLong)
case _ => throw new InterpreterException(s"Invalid syntax for @timestamp. " +
s"It should comply to the pattern ${TIMESTAMP_PATTERN.toString}")
}
}
def extractRetryPolicy(text: String): RetryPolicy = {
text match {
case RETRY_POLICIES_PATTERN(retry) => retry.trim match {
case DEFAULT_POLICY => DefaultRetryPolicy
case DOWNGRADING_CONSISTENCY_RETRY => DowngradingRetryPolicy
case FALLTHROUGH_RETRY => FallThroughRetryPolicy
case LOGGING_DEFAULT_RETRY => LoggingDefaultRetryPolicy
case LOGGING_DOWNGRADING_RETRY => LoggingDowngradingRetryPolicy
case LOGGING_FALLTHROUGH_RETRY => LoggingFallThroughRetryPolicy
}
case _ => throw new InterpreterException(s"Invalid syntax for @retryPolicy. " +
s"It should comply to the pattern ${RETRY_POLICIES_PATTERN.toString}")
}
}
def extractFetchSize(text: String): FetchSize = {
text match {
case FETCHSIZE_PATTERN(fetchSize) => FetchSize(fetchSize.trim.toInt)
case _ => throw new InterpreterException(s"Invalid syntax for @fetchSize. " +
s"It should comply to the pattern ${FETCHSIZE_PATTERN.toString}")
}
}
def extractSimpleStatement(text: String): SimpleStm = {
text match {
case SIMPLE_STATEMENT_PATTERN(statement) => SimpleStm(statement)
case _ => throw new InterpreterException(s"Invalid statement '$text'. Did you forget to add ; (semi-colon) at the end of each CQL statement ?")
}
}
def extractPreparedStatement(text: String): PrepareStm = {
text match {
case PREPARE_STATEMENT_PATTERN(name,queryString) => PrepareStm(name.trim,queryString.trim)
case _ => throw new InterpreterException(s"Invalid syntax for @prepare. " +
s"It should comply to the pattern: @prepare[prepared_statement_name]=CQL Statement (without semi-colon)")
}
}
def extractRemovePreparedStatement(text: String): RemovePrepareStm= {
text match {
case REMOVE_PREPARE_STATEMENT_PATTERN(name) => RemovePrepareStm(name.trim)
case _ => throw new InterpreterException(s"Invalid syntax for @remove_prepare. " +
s"It should comply to the pattern: @remove_prepare[prepared_statement_name]")
}
}
def extractBoundStatement(text: String): BoundStm = {
text match {
case BIND_PATTERN(name,boundValues) => BoundStm(name.trim, Option(boundValues).map(_.trim).getOrElse(""))
case _ => throw new InterpreterException("Invalid syntax for @bind. It should comply to the pattern: " +
"@bind[prepared_statement_name]=10,'jdoe','John DOE',12345,'2015-07-32 12:04:23.234' " +
"OR @bind[prepared_statement_name] with no bound value. No semi-colon")
}
}
def extractBatchType(text: String): BatchStatement.Type = {
text match {
case BATCH_PATTERN(batchType) =>
val inferredType = Option(batchType).getOrElse("LOGGED")
BatchStatement.Type.valueOf(inferredType.toUpperCase)
case _ => throw new InterpreterException(s"Invalid syntax for BEGIN BATCH. " +
s"""It should comply to the pattern: ${BATCH_PATTERN.toString}""")
}
}
def extractDescribeClusterCmd(text: String): DescribeClusterCmd = {
text match {
case DESCRIBE_CLUSTER_PATTERN() => new DescribeClusterCmd
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE CLUSTER. " +
s"""It should comply to the pattern: ${DESCRIBE_CLUSTER_PATTERN.toString}""")
}
}
def extractDescribeKeyspacesCmd(text: String): DescribeKeyspacesCmd = {
text match {
case DESCRIBE_KEYSPACES_PATTERN() => new DescribeKeyspacesCmd
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE KEYSPACES. " +
s"""It should comply to the pattern: ${DESCRIBE_KEYSPACES_PATTERN.toString}""")
}
}
def extractDescribeTablesCmd(text: String): DescribeTablesCmd = {
text match {
case DESCRIBE_TABLES_PATTERN() => new DescribeTablesCmd
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE TABLES. " +
s"""It should comply to the pattern: ${DESCRIBE_TABLES_PATTERN.toString}""")
}
}
def extractDescribeKeyspaceCmd(text: String): DescribeKeyspaceCmd = {
text match {
case DESCRIBE_KEYSPACE_PATTERN(keyspace) => new DescribeKeyspaceCmd(keyspace)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE KEYSPACE. " +
s"""It should comply to the pattern: ${DESCRIBE_KEYSPACE_PATTERN.toString}""")
}
}
def extractDescribeTableCmd(text: String): DescribeTableCmd = {
text match {
case DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN(keyspace,table) => new DescribeTableCmd(Option(keyspace),table)
case DESCRIBE_TABLE_PATTERN(table) => new DescribeTableCmd(Option.empty,table)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE TABLE. " +
s"""It should comply to the patterns: ${DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TABLE_PATTERN.toString}""".stripMargin)
}
}
def extractDescribeTypeCmd(text: String): DescribeUDTCmd = {
text match {
case DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN(keyspace,table) => new DescribeUDTCmd(Option(keyspace),table)
case DESCRIBE_TYPE_PATTERN(table) => new DescribeUDTCmd(Option.empty,table)
case _ => throw new InterpreterException(s"Invalid syntax for DESCRIBE TYPE. " +
s"""It should comply to the patterns: ${DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TYPE_PATTERN.toString}""".stripMargin)
}
}
def extractHelpCmd(text: String): HelpCmd = {
text match {
case HELP_PATTERN() => new HelpCmd
case _ => throw new InterpreterException(s"Invalid syntax for HELP. " +
s"""It should comply to the patterns: ${HELP_PATTERN.toString}""".stripMargin)
}
}
}

139
cassandra/src/main/scala/org/apache/zeppelin/cassandra/TextBlockHierarchy.scala Normal file
View file

@ -0,0 +1,139 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import com.datastax.driver.core._
/**
* Define a Scala object hierarchy
* for input text parsing
*/
object TextBlockHierarchy {
sealed trait BlockType
object ParameterBlock extends BlockType
object StatementBlock extends BlockType
object DescribeBlock extends BlockType
object CommentBlock extends BlockType
abstract class AnyBlock(val blockType: BlockType) {
def get[U <: AnyBlock]: U = {
this.asInstanceOf[U]
}
}
case class Comment(text:String) extends AnyBlock(CommentBlock)
sealed trait ParameterType
object ConsistencyParam extends ParameterType
object SerialConsistencyParam extends ParameterType
object TimestampParam extends ParameterType
object RetryPolicyParam extends ParameterType
object FetchSizeParam extends ParameterType
abstract class QueryParameters(val paramType: ParameterType) extends AnyBlock(ParameterBlock) {
def getParam[U <: QueryParameters]: U = {
this.asInstanceOf[U]
}
}
case class Consistency(value: ConsistencyLevel) extends QueryParameters(ConsistencyParam)
case class SerialConsistency(value: ConsistencyLevel) extends QueryParameters(SerialConsistencyParam)
case class Timestamp(value: Long) extends QueryParameters(TimestampParam)
case class FetchSize(value: Int) extends QueryParameters(FetchSizeParam)
abstract class RetryPolicy extends QueryParameters(RetryPolicyParam)
object DefaultRetryPolicy extends RetryPolicy
object DowngradingRetryPolicy extends RetryPolicy
object FallThroughRetryPolicy extends RetryPolicy
object LoggingDefaultRetryPolicy extends RetryPolicy
object LoggingDowngradingRetryPolicy extends RetryPolicy
object LoggingFallThroughRetryPolicy extends RetryPolicy
sealed trait StatementType
object PrepareStatementType extends StatementType
object RemovePrepareStatementType extends StatementType
object BoundStatementType extends StatementType
object SimpleStatementType extends StatementType
object BatchStatementType extends StatementType
object DescribeClusterStatementType extends StatementType
object DescribeAllKeyspacesStatementType extends StatementType
object DescribeKeyspaceStatementType extends StatementType
object DescribeAllTablesStatementType extends StatementType
object DescribeTableStatementType extends StatementType
object DescribeTypeStatementType extends StatementType
object HelpStatementType extends StatementType
abstract class QueryStatement(val statementType: StatementType) extends AnyBlock(StatementBlock) {
def getStatement[U<: QueryStatement]: U = {
this.asInstanceOf[U]
}
}
case class SimpleStm(text:String) extends QueryStatement(SimpleStatementType)
case class PrepareStm(name: String, query:String) extends QueryStatement(PrepareStatementType)
case class RemovePrepareStm(name:String) extends QueryStatement(RemovePrepareStatementType)
case class BoundStm(name: String, values:String) extends QueryStatement(BoundStatementType)
case class BatchStm(batchType: BatchStatement.Type, statements: List[QueryStatement])
extends QueryStatement(BatchStatementType)
sealed trait DescribeCommandStatement {
val statement: String
}
class DescribeClusterCmd(override val statement: String = "DESCRIBE CLUSTER;")
extends QueryStatement(DescribeClusterStatementType) with DescribeCommandStatement
class DescribeKeyspacesCmd(override val statement: String = "DESCRIBE KEYSPACES;")
extends QueryStatement(DescribeAllKeyspacesStatementType) with DescribeCommandStatement
class DescribeTablesCmd(override val statement: String = "DESCRIBE TABLES;")
extends QueryStatement(DescribeAllTablesStatementType) with DescribeCommandStatement
case class DescribeKeyspaceCmd(keyspace: String) extends QueryStatement(DescribeKeyspaceStatementType)
with DescribeCommandStatement {
override val statement: String = s"DESCRIBE KEYSPACE $keyspace;"
}
case class DescribeTableCmd(keyspace:Option[String],table: String) extends QueryStatement(DescribeTableStatementType)
with DescribeCommandStatement {
override val statement: String = keyspace match {
case Some(ks) => s"DESCRIBE TABLE $ks.$table;"
case None => s"DESCRIBE TABLE $table;"
}
}
case class DescribeUDTCmd(keyspace:Option[String],udtName: String) extends QueryStatement(DescribeTypeStatementType)
with DescribeCommandStatement {
override val statement: String = keyspace match {
case Some(ks) => s"DESCRIBE TYPE $ks.$udtName;"
case None => s"DESCRIBE TYPE $udtName;"
}
}
class HelpCmd extends QueryStatement(HelpStatementType)
}

670
cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java Normal file
View file

@ -0,0 +1,670 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra;
import static com.datastax.driver.core.ProtocolOptions.DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS;
import static com.google.common.collect.FluentIterable.from;
import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_CLUSTER_NAME;
import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_COMPRESSION_PROTOCOL;
import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_CREDENTIALS_PASSWORD;
import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_CREDENTIALS_USERNAME;
import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_HOSTS;
import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_PORT;
import static org.apache.zeppelin.cassandra.CassandraInterpreter.CASSANDRA_PROTOCOL_VERSION;
import static org.apache.zeppelin.cassandra.CassandraInterpreter.*;
import static org.assertj.core.api.Assertions.*;
import static org.mockito.Mockito.when;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.Session;
import info.archinnov.achilles.junit.AchillesResource;
import info.archinnov.achilles.junit.AchillesResourceBuilder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import scala.io.Source;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Properties;
@RunWith(MockitoJUnitRunner.class)
public class CassandraInterpreterTest {
private static final String ARTISTS_TABLE = "zeppelin.artists";
@ClassRule
public static AchillesResource resource = AchillesResourceBuilder
.noEntityPackages()
.withKeyspaceName("zeppelin")
.withScript("prepare_schema.cql")
.withScript("prepare_data.cql")
.build();
private static Session session = resource.getNativeSession();
private static CassandraInterpreter interpreter;
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext intrContext;
@BeforeClass
public static void setUp() {
Properties properties = new Properties();
final Cluster cluster = resource.getNativeSession().getCluster();
properties.setProperty(CASSANDRA_CLUSTER_NAME, cluster.getClusterName());
properties.setProperty(CASSANDRA_COMPRESSION_PROTOCOL, "NONE");
properties.setProperty(CASSANDRA_CREDENTIALS_USERNAME, "none");
properties.setProperty(CASSANDRA_CREDENTIALS_PASSWORD, "none");
properties.setProperty(CASSANDRA_PROTOCOL_VERSION, "3");
properties.setProperty(CASSANDRA_LOAD_BALANCING_POLICY, "DEFAULT");
properties.setProperty(CASSANDRA_RETRY_POLICY, "DEFAULT");
properties.setProperty(CASSANDRA_RECONNECTION_POLICY, "DEFAULT");
properties.setProperty(CASSANDRA_SPECULATIVE_EXECUTION_POLICY, "DEFAULT");
properties.setProperty(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS,
DEFAULT_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS + "");
properties.setProperty(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL, "100");
properties.setProperty(CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE, "100");
properties.setProperty(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL, "2");
properties.setProperty(CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE, "1");
properties.setProperty(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL, "8");
properties.setProperty(CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE, "2");
properties.setProperty(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL, "1024");
properties.setProperty(CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE, "256");
properties.setProperty(CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS, "120");
properties.setProperty(CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS, "5000");
properties.setProperty(CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS, "30");
properties.setProperty(CASSANDRA_QUERY_DEFAULT_CONSISTENCY, "ONE");
properties.setProperty(CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY, "SERIAL");
properties.setProperty(CASSANDRA_QUERY_DEFAULT_FETCH_SIZE, "5000");
properties.setProperty(CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS, "5000");
properties.setProperty(CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS, "12000");
properties.setProperty(CASSANDRA_SOCKET_TCP_NO_DELAY, "true");
properties.setProperty(CASSANDRA_HOSTS, from(cluster.getMetadata().getAllHosts()).first().get().getAddress().getHostAddress());
properties.setProperty(CASSANDRA_PORT, cluster.getConfiguration().getProtocolOptions().getPort()+"");
interpreter = new CassandraInterpreter(properties);
interpreter.open();
}
@AfterClass
public static void tearDown() {
interpreter.close();
}
@Before
public void prepareContext() {
when(intrContext.getParagraphTitle()).thenReturn("Paragraph1");
}
@Test
public void should_create_cluster_and_session_upon_call_to_open() throws Exception {
assertThat(interpreter.cluster).isNotNull();
assertThat(interpreter.cluster.getClusterName()).isEqualTo(resource.getNativeSession().getCluster().getClusterName());
assertThat(interpreter.session).isNotNull();
assertThat(interpreter.helper).isNotNull();
}
@Test
public void should_interpret_simple_select() throws Exception {
//Given
//When
final InterpreterResult actual = interpreter.interpret("SELECT * FROM " + ARTISTS_TABLE + " LIMIT 10;", intrContext);
//Then
assertThat(actual).isNotNull();
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("name\tborn\tcountry\tdied\tgender\tstyles\ttype\n" +
"Bogdan Raczynski\t1977-01-01\tPoland\tnull\tMale\t[Dance, Electro]\tPerson\n" +
"Krishna Das\t1947-05-31\tUSA\tnull\tMale\t[Unknown]\tPerson\n" +
"Sheryl Crow\t1962-02-11\tUSA\tnull\tFemale\t[Classic, Rock, Country, Blues, Pop, Folk]\tPerson\n" +
"Doof\t1968-08-31\tUnited Kingdom\tnull\tnull\t[Unknown]\tPerson\n" +
"House of Large Sizes\t1986-01-01\tUSA\t2003\tnull\t[Unknown]\tGroup\n" +
"Fanfarlo\t2006-01-01\tUnited Kingdom\tnull\tnull\t[Rock, Indie, Pop, Classic]\tGroup\n" +
"Jeff Beck\t1944-06-24\tUnited Kingdom\tnull\tMale\t[Rock, Pop, Classic]\tPerson\n" +
"Los Paranoias\tnull\tUnknown\tnull\tnull\t[Unknown]\tnull\n" +
"…And You Will Know Us by the Trail of Dead\t1994-01-01\tUSA\tnull\tnull\t[Rock, Pop, Classic]\tGroup\n");
}
@Test
public void should_interpret_select_statement() throws Exception {
//Given
//When
final InterpreterResult actual = interpreter.interpret("SELECT * FROM " + ARTISTS_TABLE + " LIMIT 2;", intrContext);
//Then
assertThat(actual).isNotNull();
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("name\tborn\tcountry\tdied\tgender\tstyles\ttype\n" +
"Bogdan Raczynski\t1977-01-01\tPoland\tnull\tMale\t[Dance, Electro]\tPerson\n" +
"Krishna Das\t1947-05-31\tUSA\tnull\tMale\t[Unknown]\tPerson\n");
}
@Test
public void should_interpret_multiple_statements_with_single_line_logged_batch() throws Exception {
//Given
String statements = "CREATE TABLE IF NOT EXISTS zeppelin.albums(\n" +
" title text PRIMARY KEY,\n" +
" artist text,\n" +
" year int\n" +
");\n" +
"BEGIN BATCH"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Impossible Dream EP','Carter the Unstoppable Sex Machine',1992);"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Way You Are','Tears for Fears',1983);"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('Primitive','Soulfly',2003);"+
"APPLY BATCH;\n"+
"SELECT * FROM zeppelin.albums;";
//When
final InterpreterResult actual = interpreter.interpret(statements, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("title\tartist\tyear\n" +
"The Impossible Dream EP\tCarter the Unstoppable Sex Machine\t1992\n" +
"The Way You Are\tTears for Fears\t1983\n" +
"Primitive\tSoulfly\t2003\n");
}
@Test
public void should_throw_statement_not_having_semi_colon() throws Exception {
//Given
String statement = "SELECT * zeppelin.albums";
//When
final InterpreterResult actual = interpreter.interpret(statement, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message())
.contains("Error parsing input:\n" +
"\t'SELECT * zeppelin.albums'\n" +
"Did you forget to add ; (semi-colon) at the end of each CQL statement ?");
}
@Test
public void should_validate_statement() throws Exception {
//Given
String statement = "SELECT * zeppelin.albums;";
//When
final InterpreterResult actual = interpreter.interpret(statement, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message()).contains("line 1:9 missing K_FROM at 'zeppelin' (SELECT * [zeppelin]....)");
}
@Test
public void should_execute_statement_with_consistency_option() throws Exception {
//Given
String statement = "@consistency=THREE\n" +
"SELECT * FROM zeppelin.artists LIMIT 1;";
//When
final InterpreterResult actual = interpreter.interpret(statement, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message())
.contains("Not enough replica available for query at consistency THREE (3 required but only 1 alive)");
}
@Test
public void should_execute_statement_with_serial_consistency_option() throws Exception {
//Given
String statement = "@serialConsistency=SERIAL\n" +
"SELECT * FROM zeppelin.artists LIMIT 1;";
//When
final InterpreterResult actual = interpreter.interpret(statement, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
}
@Test
public void should_execute_statement_with_timestamp_option() throws Exception {
//Given
String statement1 = "INSERT INTO zeppelin.ts(key,val) VALUES('k','v1');";
String statement2 = "@timestamp=15\n" +
"INSERT INTO zeppelin.ts(key,val) VALUES('k','v2');";
// Insert v1 with current timestamp
interpreter.interpret(statement1, intrContext);
Thread.sleep(1);
//When
// Insert v2 with past timestamp
interpreter.interpret(statement2, intrContext);
final String actual = session.execute("SELECT * FROM zeppelin.ts LIMIT 1").one().getString("val");
//Then
assertThat(actual).isEqualTo("v1");
}
@Test
public void should_execute_statement_with_retry_policy() throws Exception {
//Given
String statement = "@retryPolicy=" + interpreter.LOGGING_DOWNGRADING_RETRY + "\n" +
"@consistency=THREE\n" +
"SELECT * FROM zeppelin.artists LIMIT 1;";
//When
final InterpreterResult actual = interpreter.interpret(statement, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
}
@Test
public void should_execute_prepared_and_bound_statements() throws Exception {
//Given
String queries = "@prepare[ps]=INSERT INTO zeppelin.prepared(key,val) VALUES(?,?)\n" +
"@prepare[select]=SELECT * FROM zeppelin.prepared WHERE key=:key\n" +
"@bind[ps]='myKey','myValue'\n" +
"@bind[select]='myKey'";
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("key\tval\n" +
"myKey\tmyValue\n");
}
@Test
public void should_execute_bound_statement() throws Exception {
//Given
String queries = "@prepare[users_insert]=INSERT INTO zeppelin.users" +
"(login,firstname,lastname,addresses,location)" +
"VALUES(:login,:fn,:ln,:addresses,:loc)\n" +
"@bind[users_insert]='jdoe','John','DOE'," +
"{street_number: 3, street_name: 'Beverly Hills Bld', zip_code: 90209," +
" country: 'USA', extra_info: ['Right on the hills','Next to the post box']," +
" phone_numbers: {'home': 2016778524, 'office': 2015790847}}," +
"('USA', 90209, 'Beverly Hills')\n" +
"SELECT * FROM zeppelin.users WHERE login='jdoe';";
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo(
"login\taddresses\tage\tdeceased\tfirstname\tlast_update\tlastname\tlocation\n" +
"jdoe\t" +
"{street_number:3, street_name:'Beverly Hills Bld', zip_code:90209," +
" country:'USA', extra_info:['Right on the hills', 'Next to the post box'], " +
"phone_numbers:{'office':2015790847, 'home':2016778524}}\tnull\t" +
"null\t" +
"John\t" +
"null\t" +
"DOE\t" +
"('USA', 90209, 'Beverly Hills')\n");
}
@Test
public void should_exception_when_executing_unknown_bound_statement() throws Exception {
//Given
String queries = "@bind[select_users]='jdoe'";
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message())
.isEqualTo("The statement 'select_users' can not be bound to values. " +
"Are you sure you did prepare it with @prepare[select_users] ?");
}
@Test
public void should_extract_variable_from_statement() throws Exception {
//Given
when(intrContext.getGui().input("login", "hsue")).thenReturn("hsue");
when(intrContext.getGui().input("age", "27")).thenReturn("27");
String queries = "@prepare[test_insert_with_variable]=" +
"INSERT INTO zeppelin.users(login,firstname,lastname,age) VALUES(?,?,?,?)\n" +
"@bind[test_insert_with_variable]='{{login=hsue}}','Helen','SUE',{{age=27}}\n" +
"SELECT firstname,lastname,age FROM zeppelin.users WHERE login='hsue';";
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("firstname\tlastname\tage\n" +
"Helen\tSUE\t27\n");
}
@Test
public void should_just_prepare_statement() throws Exception {
//Given
String queries = "@prepare[just_prepare]=SELECT name,country,styles FROM zeppelin.artists LIMIT 3";
final String expected = reformatHtml(
readTestResource("/scalate/NoResult.html"));
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_execute_bound_statement_with_no_bound_value() throws Exception {
//Given
String queries = "@prepare[select_no_bound_value]=SELECT name,country,styles FROM zeppelin.artists LIMIT 3\n" +
"@bind[select_no_bound_value]";
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("name\tcountry\tstyles\n" +
"Bogdan Raczynski\tPoland\t[Dance, Electro]\n" +
"Krishna Das\tUSA\t[Unknown]\n" +
"Sheryl Crow\tUSA\t[Classic, Rock, Country, Blues, Pop, Folk]\n");
}
@Test
public void should_parse_date_value() throws Exception {
//Given
String queries = "@prepare[parse_date]=INSERT INTO zeppelin.users(login,last_update) VALUES(?,?)\n" +
"@bind[parse_date]='last_update','2015-07-30 12:00:01'\n" +
"SELECT last_update FROM zeppelin.users WHERE login='last_update';";
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).contains("last_update\n" +
"Thu Jul 30 12:00:01");
}
@Test
public void should_bind_null_value() throws Exception {
//Given
String queries = "@prepare[bind_null]=INSERT INTO zeppelin.users(login,firstname,lastname) VALUES(?,?,?)\n" +
"@bind[bind_null]='bind_null',null,'NULL'\n" +
"SELECT firstname,lastname FROM zeppelin.users WHERE login='bind_null';";
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("firstname\tlastname\n" +
"null\tNULL\n");
}
@Test
public void should_bind_boolean_value() throws Exception {
//Given
String queries = "@prepare[bind_boolean]=INSERT INTO zeppelin.users(login,deceased) VALUES(?,?)\n" +
"@bind[bind_boolean]='bind_bool',false\n" +
"SELECT login,deceased FROM zeppelin.users WHERE login='bind_bool';";
//When
final InterpreterResult actual = interpreter.interpret(queries, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(actual.message()).isEqualTo("login\tdeceased\n" +
"bind_bool\tfalse\n");
}
@Test
public void should_fail_when_executing_a_removed_prepared_statement() throws Exception {
//Given
String prepare_first = "@prepare[to_be_removed]=INSERT INTO zeppelin.users(login,deceased) VALUES(?,?)";
interpreter.interpret(prepare_first, intrContext);
String remove_prepared = "@remove_prepare[to_be_removed]\n" +
"@bind[to_be_removed]='bind_bool'";
//When
final InterpreterResult actual = interpreter.interpret(remove_prepared, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message()).isEqualTo("The statement 'to_be_removed' can not be bound to values. " +
"Are you sure you did prepare it with @prepare[to_be_removed] ?");
}
@Test
public void should_display_statistics_for_non_select_statement() throws Exception {
//Given
String query = "USE zeppelin;\nCREATE TABLE IF NOT EXISTS no_select(id int PRIMARY KEY);";
final String rawResult = reformatHtml(readTestResource("/scalate/NoResultWithExecutionInfo.html"));
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
final Cluster cluster = session.getCluster();
final int port = cluster.getConfiguration().getProtocolOptions().getPort();
final String address = cluster.getMetadata().getAllHosts().iterator().next()
.getAddress().getHostAddress()
.replaceAll("/", "").replaceAll("\\[", "").replaceAll("\\]","");
//Then
final String expected = rawResult.replaceAll("TRIED_HOSTS", address+":"+port)
.replaceAll("QUERIED_HOSTS", address +":"+port);
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_error_and_display_stack_trace() throws Exception {
//Given
String query = "@consistency=THREE\n" +
"SELECT * FROM zeppelin.users LIMIT 3;";
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message()).contains("All host(s) tried for query failed");
}
@Test
public void should_describe_cluster() throws Exception {
//Given
String query = "DESCRIBE CLUSTER;";
final String expected = reformatHtml(
readTestResource("/scalate/DescribeCluster.html"));
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_describe_keyspaces() throws Exception {
//Given
String query = "DESCRIBE KEYSPACES;";
final String expected = reformatHtml(
readTestResource("/scalate/DescribeKeyspaces.html"));
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_describe_keyspace() throws Exception {
//Given
String query = "DESCRIBE KEYSPACE live_data;";
final String expected = reformatHtml(
readTestResource("/scalate/DescribeKeyspace_live_data.html"));
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_describe_table() throws Exception {
//Given
String query = "DESCRIBE TABLE live_data.complex_table;";
final String expected = reformatHtml(
readTestResource("/scalate/DescribeTable_live_data_complex_table.html"));
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_describe_udt() throws Exception {
//Given
String query = "DESCRIBE TYPE live_data.address;";
final String expected = reformatHtml(
readTestResource("/scalate/DescribeType_live_data_address.html"));
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_describe_udt_withing_logged_in_keyspace() throws Exception {
//Given
String query = "USE live_data;\n" +
"DESCRIBE TYPE address;";
final String expected = reformatHtml(
readTestResource("/scalate/DescribeType_live_data_address_within_current_keyspace.html"));
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
@Test
public void should_error_describing_non_existing_table() throws Exception {
//Given
String query = "USE system;\n" +
"DESCRIBE TABLE complex_table;";
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message()).contains("Cannot find table system.complex_table");
}
@Test
public void should_error_describing_non_existing_udt() throws Exception {
//Given
String query = "USE system;\n" +
"DESCRIBE TYPE address;";
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.ERROR);
assertThat(actual.message()).contains("Cannot find type system.address");
}
@Test
public void should_show_help() throws Exception {
//Given
String query = "HELP;";
final String expected = reformatHtml(readTestResource("/scalate/Help.html"));
//When
final InterpreterResult actual = interpreter.interpret(query, intrContext);
//Then
assertThat(actual.code()).isEqualTo(Code.SUCCESS);
assertThat(reformatHtml(actual.message())).isEqualTo(expected);
}
private static String reformatHtml(String rawHtml) {
return rawHtml
.replaceAll("\\s*\n\\s*","")
.replaceAll(">\\s+<", "><")
.replaceAll("(?s)data-target=\"#[a-f0-9-]+(?:_asCQL)?\"", "")
.replaceAll("(?s)id=\"[a-f0-9-]+(?:_asCQL)?\"", "")
.trim();
}
private static String readTestResource(String testResource) {
StringBuilder builder = new StringBuilder();
InputStream stream = testResource.getClass().getResourceAsStream(testResource);
try (BufferedReader br = new BufferedReader(new InputStreamReader(stream))) {
String line;
while ((line = br.readLine()) != null) {
builder.append(line).append("\n");
}
} catch (Exception ex) {
throw new RuntimeException(ex);
}
return builder.toString();
}
}

311
cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java Normal file
View file

@ -0,0 +1,311 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra;
import static com.datastax.driver.core.BatchStatement.Type.UNLOGGED;
import static com.datastax.driver.core.ConsistencyLevel.ALL;
import static com.datastax.driver.core.ConsistencyLevel.LOCAL_SERIAL;
import static com.datastax.driver.core.ConsistencyLevel.ONE;
import static com.datastax.driver.core.ConsistencyLevel.QUORUM;
import static com.datastax.driver.core.ConsistencyLevel.SERIAL;
import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.*;
import static org.mockito.Mockito.*;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.Statement;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.display.Input.ParamOption;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import scala.Option;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.*;
@RunWith(MockitoJUnitRunner.class)
public class InterpreterLogicTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext intrContext;
@Mock
private Session session;
final InterpreterLogic helper = new InterpreterLogic(session);
@Captor
ArgumentCaptor<ParamOption[]> optionsCaptor;
@Test
public void should_parse_input_string_block() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10;";
//When
final List<AnyBlock> anyBlocks = this.<AnyBlock>toJavaList(helper.parseInput(input));
//Then
assertThat(anyBlocks).hasSize(1);
assertThat(anyBlocks.get(0)).isInstanceOf(SimpleStm.class);
}
@Test
public void should_exception_while_parsing_input() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10";
//When
expectedException.expect(InterpreterException.class);
expectedException.expectMessage("Error parsing input:\n" +
"\t'SELECT * FROM users LIMIT 10'\n" +
"Did you forget to add ; (semi-colon) at the end of each CQL statement ?");
helper.parseInput(input);
}
@Test
public void should_extract_variable_and_default_value() throws Exception {
//Given
when(intrContext.getGui().input("table", "zeppelin.demo")).thenReturn("zeppelin.demo");
when(intrContext.getGui().input("id", "'John'")).thenReturn("'John'");
//When
final String actual = helper.maybeExtractVariables("SELECT * FROM {{table=zeppelin.demo}} WHERE id={{id='John'}}", intrContext);
//Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='John'");
}
@Test
public void should_extract_variable_and_choices() throws Exception {
//Given
when(intrContext.getGui().select(eq("name"), eq("'Paul'"), optionsCaptor.capture())).thenReturn("'Jack'");
//When
final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.artists WHERE name={{name='Paul'|'Jack'|'Smith'}}", intrContext);
//Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.artists WHERE name='Jack'");
final List<ParamOption> paramOptions = asList(optionsCaptor.getValue());
assertThat(paramOptions.get(0).getValue()).isEqualTo("'Paul'");
assertThat(paramOptions.get(1).getValue()).isEqualTo("'Jack'");
assertThat(paramOptions.get(2).getValue()).isEqualTo("'Smith'");
}
@Test
public void should_extract_no_variable() throws Exception {
//Given
GUI gui = mock(GUI.class);
when(intrContext.getGui()).thenReturn(gui);
//When
final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.demo", intrContext);
//Then
verifyZeroInteractions(gui);
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo");
}
@Test
public void should_error_if_incorrect_variable_definition() throws Exception {
//Given
//When
expectedException.expect(ParsingException.class);
expectedException.expectMessage("Invalid bound variable definition for '{{table?zeppelin.demo}}' in 'SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}'. It should be of form 'variable=defaultValue'");
//Then
helper.maybeExtractVariables("SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}", intrContext);
}
@Test
public void should_extract_consistency_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new Consistency(ALL), new Consistency(ONE));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
assertThat(actual.consistency().get()).isEqualTo(ALL);
}
@Test
public void should_extract_serial_consistency_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new SerialConsistency(SERIAL), new SerialConsistency(LOCAL_SERIAL));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
assertThat(actual.serialConsistency().get()).isEqualTo(SERIAL);
}
@Test
public void should_extract_timestamp_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new Timestamp(123L), new Timestamp(456L));
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
assertThat(actual.timestamp().get()).isEqualTo(123L);
}
@Test
public void should_extract_retry_policy_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(DowngradingRetryPolicy$.MODULE$, LoggingDefaultRetryPolicy$.MODULE$);
//When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
assertThat(actual.retryPolicy().get()).isSameAs(DowngradingRetryPolicy$.MODULE$);
}
@Test
public void should_generate_simple_statement() throws Exception {
//Given
String input = "SELECT * FROM users LIMIT 10;";
CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM),
Option.<ConsistencyLevel>empty(),
Option.empty(),
Option.<RetryPolicy>empty(),
Option.empty());
//When
final SimpleStatement actual = helper.generateSimpleStatement(new SimpleStm(input), options, intrContext);
//Then
assertThat(actual).isNotNull();
assertThat(actual.getQueryString()).isEqualTo("SELECT * FROM users LIMIT 10;");
assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM);
}
@Test
public void should_generate_batch_statement() throws Exception {
//Given
Statement st1 = new SimpleStatement("SELECT * FROM users LIMIT 10;");
Statement st2 = new SimpleStatement("INSERT INTO users(id) VALUES(10);");
Statement st3 = new SimpleStatement("UPDATE users SET name = 'John DOE' WHERE id=10;");
CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM),
Option.<ConsistencyLevel>empty(),
Option.empty(),
Option.<RetryPolicy>empty(),
Option.empty());
//When
BatchStatement actual = helper.generateBatchStatement(UNLOGGED, options, toScalaList(asList(st1, st2, st3)));
//Then
assertThat(actual).isNotNull();
final List<Statement> statements = new ArrayList<>(actual.getStatements());
assertThat(statements).hasSize(3);
assertThat(statements.get(0)).isSameAs(st1);
assertThat(statements.get(1)).isSameAs(st2);
assertThat(statements.get(2)).isSameAs(st3);
assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM);
}
@Test
public void should_parse_bound_values() throws Exception {
//Given
String bs="'jdoe',32,'John DOE',null, true, '2014-06-12 34:00:34'";
//When
final List<String> actual = this.<String>toJavaList(helper.parseBoundValues("ps", bs));
//Then
assertThat(actual).containsExactly("'jdoe'", "32", "'John DOE'",
"null", "true", "2014-06-12 34:00:34");
}
@Test
public void should_parse_simple_date() throws Exception {
//Given
String dateString = "2015-07-30 12:00:01";
//When
final Date actual = helper.parseDate(dateString);
//Then
Calendar calendar = Calendar.getInstance();
calendar.setTime(actual);
assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015);
assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY);
assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30);
assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12);
assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0);
assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1);
}
@Test
public void should_parse_accurate_date() throws Exception {
//Given
String dateString = "2015-07-30 12:00:01.123";
//When
final Date actual = helper.parseDate(dateString);
//Then
Calendar calendar = Calendar.getInstance();
calendar.setTime(actual);
assertThat(calendar.get(Calendar.YEAR)).isEqualTo(2015);
assertThat(calendar.get(Calendar.MONTH)).isEqualTo(Calendar.JULY);
assertThat(calendar.get(Calendar.DAY_OF_MONTH)).isEqualTo(30);
assertThat(calendar.get(Calendar.HOUR_OF_DAY)).isEqualTo(12);
assertThat(calendar.get(Calendar.MINUTE)).isEqualTo(0);
assertThat(calendar.get(Calendar.SECOND)).isEqualTo(1);
assertThat(calendar.get(Calendar.MILLISECOND)).isEqualTo(123);
}
private <A> scala.collection.immutable.List<A> toScalaList(java.util.List<A> list) {
return scala.collection.JavaConverters.asScalaBufferConverter(list).asScala().toList();
}
private <A> java.util.List<A> toJavaList(scala.collection.immutable.List<A> list){
return scala.collection.JavaConverters.seqAsJavaListConverter(list).asJava();
}
}

80
cassandra/src/test/resources/log4j.xml Normal file
View file

@ -0,0 +1,80 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright (C) 2012-2014 DuyHai DOAN
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<!DOCTYPE log4j:configuration PUBLIC
"-//APACHE//DTD LOG4J 1.2//EN"
"http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/xml/doc-files/log4j.dtd">
<log4j:configuration debug="false"
xmlns:log4j="http://jakarta.apache.org/log4j/">
<!-- Appenders -->
<appender name="ConsoleAppender"
class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern"
value="%-5p [%d{ABSOLUTE}][%x] %c@:%M %m %n"/>
</layout>
</appender>
<logger name="ACHILLES_DDL_SCRIPT">
<level value="DEBUG"/>
</logger>
<logger name="ACHILLES_DML_STATEMENT">
<level value="DEBUG"/>
</logger>
<logger name="info.archinnov.achilles">
<level value="INFO"/>
</logger>
<logger name="org.apache.zeppelin.cassandra">
<level value="DEBUG"/>
</logger>
<!-- lots of warning for embedded Cassandra, shut them up -->
<logger name="org.apache.cassandra.cql3.QueryProcessor">
<level value="OFF"/>
</logger>
<logger name="org.apache.cassandra.service.CassandraDaemon">
<level value="OFF"/>
</logger>
<logger name="com.datastax.driver.core.ControlConnection">
<level value="OFF"/>
</logger>
<logger name="org.apache.cassandra.db.Memtable">
<level value="OFF"/>
</logger>
<logger name="org.apache.cassandra.utils.CLibrary">
<level value="OFF"/>
</logger>
<logger name="com.datastax.driver.core.NettyUtil">
<level value="OFF"/>
</logger>
<logger name="io.netty">
<level value="OFF"/>
</logger>
<!-- ********************************************************* -->
<!-- Root Loggers -->
<!-- ********************************************************* -->
<root>
<level value="WARN"/>
<appender-ref ref="ConsoleAppender"/>
</root>
</log4j:configuration>

18
cassandra/src/test/resources/prepare_data.cql Normal file
View file

@ -0,0 +1,18 @@
TRUNCATE zeppelin.artists;
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('Bogdan Raczynski','1977-01-01','Poland',null,'Male',['Dance', 'Electro'],'Person');
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('Krishna Das','1947-05-31','USA',null,'Male',['Unknown'],'Person');
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('Sheryl Crow','1962-02-11','USA',null,'Female',['Classic', 'Rock', 'Country', 'Blues', 'Pop', 'Folk'],'Person');
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('Doof','1968-08-31','United Kingdom',null,null,['Unknown'],'Person');
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('House of Large Sizes','1986-01-01','USA','2003',null,['Unknown'],'Group');
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('Fanfarlo','2006-01-01','United Kingdom',null,null,['Rock', 'Indie', 'Pop', 'Classic'],'Group');
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('Jeff Beck','1944-06-24','United Kingdom',null,'Male',['Rock', 'Pop', 'Classic'],'Person');
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('Los Paranoias',null,'Unknown',null,null,['Unknown'],null);
INSERT INTO zeppelin.artists(name,born,country,died,gender,styles,type) VALUES('…And You Will Know Us by the Trail of Dead','1994-01-01','USA',null,null,['Rock', 'Pop', 'Classic'],'Group');
TRUNCATE zeppelin.ts;
TRUNCATE zeppelin.prepared;
TRUNCATE zeppelin.users;

114
cassandra/src/test/resources/prepare_schema.cql Normal file
View file

@ -0,0 +1,114 @@
CREATE KEYSPACE IF NOT EXISTS zeppelin
WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':1}
AND DURABLE_WRITES=false;
CREATE TABLE IF NOT EXISTS zeppelin.artists(
name text PRIMARY KEY,
born text,
died text,
country text,
gender text,
type text,
styles list<text>
);
CREATE TABLE IF NOT EXISTS zeppelin.ts(
key text PRIMARY KEY,
val text
);
CREATE TABLE IF NOT EXISTS zeppelin.prepared(
key text PRIMARY KEY,
val text
);
CREATE TYPE IF NOT EXISTS zeppelin.address(
street_number int,
street_name text,
zip_code int,
country text,
extra_info list<text>,
phone_numbers map<text,bigint>
);
CREATE TABLE IF NOT EXISTS zeppelin.users(
login text,
firstname text,
lastname text,
age int,
deceased boolean,
last_update timestamp,
addresses frozen<address>,
location frozen<tuple<text,bigint,text>>,
PRIMARY KEY(login)
);
CREATE KEYSPACE IF NOT EXISTS samples
WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':1}
AND DURABLE_WRITES=false;
CREATE KEYSPACE IF NOT EXISTS live_data
WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':1}
AND DURABLE_WRITES=false;
CREATE TYPE IF NOT EXISTS live_data.address (
number int,
street text,
zip int,
city text,
country text
);
CREATE TABLE IF NOT EXISTS live_data.complex_table(
pk1 uuid,
pk2 int,
my_static1 text static,
my_static2 text static,
clustering1 timestamp,
clustering2 int,
clustering3 text,
indexed1 text,
indexed2 int,
simple double,
my_list list<text>,
my_udt_list frozen<list<address>>,
my_udt frozen<address>,
my_map map<int,text>,
key_indexed_map map<int,text>,
entries_indexed_map map<int,text>,
PRIMARY KEY((pk1,pk2),clustering1, clustering2, clustering3)
) WITH CLUSTERING ORDER BY (clustering1 DESC, clustering2 ASC, clustering3 DESC);
CREATE INDEX IF NOT EXISTS pk2idx ON live_data.complex_table(pk2);
CREATE INDEX IF NOT EXISTS clustering2idx ON live_data.complex_table(clustering2);
CREATE INDEX IF NOT EXISTS idx1 ON live_data.complex_table(indexed1);
CREATE INDEX IF NOT EXISTS idx2 ON live_data.complex_table(indexed2);
CREATE INDEX IF NOT EXISTS keys_map_idx ON live_data.complex_table(KEYS(key_indexed_map));
CREATE TABLE IF NOT EXISTS live_data.sensor_data (
sensor_id uuid,
month int,
provider text static,
model_number text static,
characteristics map<text, text> static,
date timestamp,
value double,
PRIMARY KEY((sensor_id, month), date)
) WITH CLUSTERING ORDER BY (date DESC);
CREATE TYPE IF NOT EXISTS live_data.geolocation (
latitude double,
longitude double
);
CREATE TABLE IF NOT EXISTS live_data.stations (
station_id uuid,
sensors frozen<map<uuid,geolocation>>,
PRIMARY KEY (station_id)
);

98
cassandra/src/test/resources/scalate/DescribeCluster.html Normal file
View file

@ -0,0 +1,98 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li>
<a><strong>DESCRIBE CLUSTER;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="row">
<div class="col-md-4"></div>
<div class="col-md-4 col-offset-md-4">
<div class="table-responsive table-bordered">
<table class="table">
<caption>
<h4 class="text-muted">
<i class="glyphicon glyphicon-dashboard"/>&nbsp;Test Cluster
</h4>
</caption>
<thead>
<tr>
<th>Partitioner</th>
</tr>
</thead>
<tbody>
<tr>
<td>org.apache.cassandra.dht.Murmur3Partitioner</td>
</tr>
<tbody>
</table>
</div>
</div>
</div>

41
cassandra/src/test/resources/scalate/DescribeKeyspace_alone.html Normal file
View file

@ -0,0 +1,41 @@
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<caption>
<h4 class="text-danger">
<i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;live_data
</h4>
</caption>
<thead>
<tr>
<th class="col-md-10">Replication</th>
<th class="col-md-2">Durable Writes</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-10">{'replication_factor' : '1', 'class' : 'org.apache.cassandra.locator.SimpleStrategy'}</td>
<td class="col-md-2">false</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#3b294060-3516-11e5-ab36-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="3b294060-3516-11e5-ab36-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE KEYSPACE live_data WITH REPLICATION = {
'class' : 'org.apache.cassandra.locator.SimpleStrategy',
'replication_factor': '1' }
AND DURABLE_WRITES = false;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>

819
cassandra/src/test/resources/scalate/DescribeKeyspace_live_data.html Normal file
View file

@ -0,0 +1,819 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li role="presentation" class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;<strong>live_data</strong></span>
<span class="text-danger caret"></span>
<ul class="dropdown-menu">
<li class="dropdown-header"><span class="text-primary">Tables</span></li>
<li>
<a role="button" data-toggle="collapse" data-target="#91e7ac20-3516-11e5-9d5f-8f0ea8ae1a37">
<span class="text-primary"><i class="glyphicon glyphicon-th-list"/>&nbsp;complex_table</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#93e1a760-3516-11e5-9d5f-8f0ea8ae1a37">
<span class="text-primary"><i class="glyphicon glyphicon-th-list"/>&nbsp;sensor_data</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#93e306f0-3516-11e5-9d5f-8f0ea8ae1a37">
<span class="text-primary"><i class="glyphicon glyphicon-th-list"/>&nbsp;stations</span>
</a>
</li>
<li role="separator" class="divider text-muted"></li>
<li class="dropdown-header"><span class="text-warning">User Defined Types</span></li>
<li>
<a role="button" data-toggle="collapse" data-target="#93e48d90-3516-11e5-9d5f-8f0ea8ae1a37">
<span class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;address</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#942c9310-3516-11e5-9d5f-8f0ea8ae1a37">
<span class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;geolocation</span>
</a>
</li>
</ul>
</a>
</li>
<li>
<a><strong>DESCRIBE KEYSPACE live_data;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="container">
<!-- Keyspace -->
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-10">Replication</th>
<th class="col-md-2">Durable Writes</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-10">{'replication_factor' : '1', 'class' : 'org.apache.cassandra.locator.SimpleStrategy'}</td>
<td class="col-md-2">false</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#8db09220-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="8db09220-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE KEYSPACE live_data WITH REPLICATION = {
'class' : 'org.apache.cassandra.locator.SimpleStrategy',
'replication_factor': '1' }
AND DURABLE_WRITES = false;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
<!-- Tables -->
<div class="row"></div>
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-primary">Tables</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#91e7ac20-3516-11e5-9d5f-8f0ea8ae1a37" aria-expanded="false">
<span class="text-primary">
<i class="glyphicon glyphicon-th-list"/>&nbsp;complex_table
</span>
</a>
</h4>
</div>
<div id="91e7ac20-3516-11e5-9d5f-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-4">Column Type</th>
<th class="col-md-4">Column Name</th>
<th class="col-md-4">Data Type</th>
</tr>
</thead>
<tbody>
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
<span></span>
</td>
<td class="col-md-4">pk1</td>
<td class="col-md-4">uuid</td>
</tr>
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>pk2idx</em> <strong></strong>
</td>
<td class="col-md-4">pk2</td>
<td class="col-md-4">int</td>
</tr>
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
<span></span>
</td>
<td class="col-md-4">my_static1</td>
<td class="col-md-4">text</td>
</tr>
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
<span></span>
</td>
<td class="col-md-4">my_static2</td>
<td class="col-md-4">text</td>
</tr>
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes-alt" title="Sort DESC"/>
<span></span>
</td>
<td class="col-md-4">clustering1</td>
<td class="col-md-4">timestamp</td>
</tr>
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes" title="Sort ASC"/>
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>clustering2idx</em> <strong></strong>
</td>
<td class="col-md-4">clustering2</td>
<td class="col-md-4">int</td>
</tr>
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes-alt" title="Sort DESC"/>
<span></span>
</td>
<td class="col-md-4">clustering3</td>
<td class="col-md-4">text</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">entries_indexed_map</td>
<td class="col-md-4">map&lt;int, text&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>idx1</em> <strong></strong>
</td>
<td class="col-md-4">indexed1</td>
<td class="col-md-4">text</td>
</tr>
<tr>
<td class="col-md-4">
<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>idx2</em> <strong></strong>
</td>
<td class="col-md-4">indexed2</td>
<td class="col-md-4">int</td>
</tr>
<tr>
<td class="col-md-4">
<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>keys_map_idx</em> <strong></strong>
</td>
<td class="col-md-4">key_indexed_map</td>
<td class="col-md-4">map&lt;int, text&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">my_list</td>
<td class="col-md-4">list&lt;text&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">my_map</td>
<td class="col-md-4">map&lt;int, text&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">my_udt</td>
<td class="col-md-4">frozen&lt;live_data.address&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">my_udt_list</td>
<td class="col-md-4">frozen&lt;list&lt;frozen&lt;live_data.address&gt;&gt;&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">simple</td>
<td class="col-md-4">double</td>
</tr>
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#91eb0780-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="91eb0780-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TABLE live_data.complex_table (
pk1 uuid,
pk2 int,
clustering1 timestamp,
clustering2 int,
clustering3 text,
entries_indexed_map map&lt;int, text&gt;,
indexed1 text,
indexed2 int,
key_indexed_map map&lt;int, text&gt;,
my_list list&lt;text&gt;,
my_map map&lt;int, text&gt;,
my_static1 text static,
my_static2 text static,
my_udt frozen&lt;live_data.address&gt;,
my_udt_list frozen&lt;list&lt;frozen&lt;live_data.address&gt;&gt;&gt;,
simple double,
PRIMARY KEY ((pk1, pk2), clustering1, clustering2, clustering3)
) WITH CLUSTERING ORDER BY (clustering1 DESC, clustering2 ASC, clustering3 DESC)
AND read_repair_chance = 0.0
AND dclocal_read_repair_chance = 0.1
AND gc_grace_seconds = 864000
AND bloom_filter_fp_chance = 0.01
AND caching = { 'keys' : 'ALL', 'rows_per_partition' : 'NONE' }
AND comment = ''
AND compaction = { 'class' : 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy' }
AND compression = { 'sstable_compression' : 'org.apache.cassandra.io.compress.LZ4Compressor' }
AND default_time_to_live = 0
AND speculative_retry = '99.0PERCENTILE'
AND min_index_interval = 128
AND max_index_interval = 2048;
CREATE INDEX pk2idx ON live_data.complex_table (pk2);
CREATE INDEX clustering2idx ON live_data.complex_table (clustering2);
CREATE INDEX idx1 ON live_data.complex_table (indexed1);
CREATE INDEX idx2 ON live_data.complex_table (indexed2);
CREATE INDEX keys_map_idx ON live_data.complex_table (KEYS(key_indexed_map));</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#93e1a760-3516-11e5-9d5f-8f0ea8ae1a37" aria-expanded="false">
<span class="text-primary">
<i class="glyphicon glyphicon-th-list"/>&nbsp;sensor_data
</span>
</a>
</h4>
</div>
<div id="93e1a760-3516-11e5-9d5f-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-4">Column Type</th>
<th class="col-md-4">Column Name</th>
<th class="col-md-4">Data Type</th>
</tr>
</thead>
<tbody>
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
<span></span>
</td>
<td class="col-md-4">sensor_id</td>
<td class="col-md-4">uuid</td>
</tr>
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
<span></span>
</td>
<td class="col-md-4">month</td>
<td class="col-md-4">int</td>
</tr>
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
<span></span>
</td>
<td class="col-md-4">characteristics</td>
<td class="col-md-4">map&lt;text, text&gt;</td>
</tr>
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
<span></span>
</td>
<td class="col-md-4">model_number</td>
<td class="col-md-4">text</td>
</tr>
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
<span></span>
</td>
<td class="col-md-4">provider</td>
<td class="col-md-4">text</td>
</tr>
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes-alt" title="Sort DESC"/>
<span></span>
</td>
<td class="col-md-4">date</td>
<td class="col-md-4">timestamp</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">value</td>
<td class="col-md-4">double</td>
</tr>
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#93e1a761-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="93e1a761-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TABLE live_data.sensor_data (
sensor_id uuid,
month int,
date timestamp,
characteristics map&lt;text, text&gt; static,
model_number text static,
provider text static,
value double,
PRIMARY KEY ((sensor_id, month), date)
) WITH CLUSTERING ORDER BY (date DESC)
AND read_repair_chance = 0.0
AND dclocal_read_repair_chance = 0.1
AND gc_grace_seconds = 864000
AND bloom_filter_fp_chance = 0.01
AND caching = { 'keys' : 'ALL', 'rows_per_partition' : 'NONE' }
AND comment = ''
AND compaction = { 'class' : 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy' }
AND compression = { 'sstable_compression' : 'org.apache.cassandra.io.compress.LZ4Compressor' }
AND default_time_to_live = 0
AND speculative_retry = '99.0PERCENTILE'
AND min_index_interval = 128
AND max_index_interval = 2048;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#93e306f0-3516-11e5-9d5f-8f0ea8ae1a37" aria-expanded="false">
<span class="text-primary">
<i class="glyphicon glyphicon-th-list"/>&nbsp;stations
</span>
</a>
</h4>
</div>
<div id="93e306f0-3516-11e5-9d5f-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-4">Column Type</th>
<th class="col-md-4">Column Name</th>
<th class="col-md-4">Data Type</th>
</tr>
</thead>
<tbody>
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
<span></span>
</td>
<td class="col-md-4">station_id</td>
<td class="col-md-4">uuid</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">sensors</td>
<td class="col-md-4">frozen&lt;map&lt;uuid, frozen&lt;live_data.geolocation&gt;&gt;&gt;</td>
</tr>
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#93e306f1-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="93e306f1-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TABLE live_data.stations (
station_id uuid,
sensors frozen&lt;map&lt;uuid, frozen&lt;live_data.geolocation&gt;&gt;&gt;,
PRIMARY KEY (station_id)
) WITH read_repair_chance = 0.0
AND dclocal_read_repair_chance = 0.1
AND gc_grace_seconds = 864000
AND bloom_filter_fp_chance = 0.01
AND caching = { 'keys' : 'ALL', 'rows_per_partition' : 'NONE' }
AND comment = ''
AND compaction = { 'class' : 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy' }
AND compression = { 'sstable_compression' : 'org.apache.cassandra.io.compress.LZ4Compressor' }
AND default_time_to_live = 0
AND speculative_retry = '99.0PERCENTILE'
AND min_index_interval = 128
AND max_index_interval = 2048;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- UDTs -->
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-warning">User Defined Types</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#93e48d90-3516-11e5-9d5f-8f0ea8ae1a37" aria-expanded="false">
<span class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;address</span>
</a>
</h4>
</div>
<div id="93e48d90-3516-11e5-9d5f-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-6">Column Name</th>
<th class="col-md-6">Data Type</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-6">number</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">street</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">zip</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">city</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">country</td>
<td class="col-md-6">text</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#93e529d0-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="93e529d0-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TYPE live_data.address (
number int,
street text,
zip int,
city text,
country text
);</pre>
</div>
</div>
</div>
</div>
<div class="col-md-3"></div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#942c9310-3516-11e5-9d5f-8f0ea8ae1a37" aria-expanded="false">
<span class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;geolocation</span>
</a>
</h4>
</div>
<div id="942c9310-3516-11e5-9d5f-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-6">Column Name</th>
<th class="col-md-6">Data Type</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-6">latitude</td>
<td class="col-md-6">double</td>
</tr>
<tr>
<td class="col-md-6">longitude</td>
<td class="col-md-6">double</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#942c9311-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="942c9311-3516-11e5-9d5f-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TYPE live_data.geolocation (
latitude double,
longitude double
);</pre>
</div>
</div>
</div>
</div>
<div class="col-md-3"></div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>

400
cassandra/src/test/resources/scalate/DescribeKeyspaces.html Normal file
View file

@ -0,0 +1,400 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li role="presentation" class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-muted"><i class="glyphicon glyphicon-dashboard"/>&nbsp;<strong>Test Cluster</strong></span>
<span class="text-muted caret"></span>
<ul class="dropdown-menu">
<li class="dropdown-header"><span class="text-danger">Keyspaces</span></li>
<li>
<a role="button" data-toggle="collapse" data-target="#8f72f0b0-3518-11e5-aee3-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;live_data</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#8fb85e20-3518-11e5-aee3-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;samples</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#8fb996a0-3518-11e5-aee3-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;system</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#8fb9e4c0-3518-11e5-aee3-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;system_traces</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#8fba32e0-3518-11e5-aee3-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;zeppelin</span>
</a>
</li>
</ul>
</a>
</li>
<li>
<a><strong>DESCRIBE KEYSPACES;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="container">
<!-- Cluster -->
<div class="row">
<div class="col-md-4"></div>
<div class="col-md-4 col-offset-md-4">
<div class="table-responsive table-bordered">
<table class="table">
<caption>
<h4 class="text-muted">
<i class="glyphicon glyphicon-dashboard"/>&nbsp;Test Cluster
</h4>
</caption>
<thead>
<tr>
<th>Partitioner</th>
</tr>
</thead>
<tbody>
<tr>
<td>org.apache.cassandra.dht.Murmur3Partitioner</td>
</tr>
<tbody>
</table>
</div>
</div>
</div>
<div class="row"></div>
<!-- Keyspaces -->
<table width="100%">
<td><hr /></td>
<td style="width:1px; padding: 0 10px; white-space: nowrap;"><strong class="text-danger">Keyspaces</strong></td>
<td><hr /></td>
</table>
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#8f72f0b0-3518-11e5-aee3-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;live_data</span>
</a>
</h4>
</div>
<div id="8f72f0b0-3518-11e5-aee3-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-10">Replication</th>
<th class="col-md-2">Durable Writes</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-10">{'replication_factor' : '1', 'class' : 'org.apache.cassandra.locator.SimpleStrategy'}</td>
<td class="col-md-2">false</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#8f738cf0-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="8f738cf0-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE KEYSPACE live_data WITH REPLICATION = {
'class' : 'org.apache.cassandra.locator.SimpleStrategy',
'replication_factor': '1' }
AND DURABLE_WRITES = false;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#8fb85e20-3518-11e5-aee3-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;samples</span>
</a>
</h4>
</div>
<div id="8fb85e20-3518-11e5-aee3-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-10">Replication</th>
<th class="col-md-2">Durable Writes</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-10">{'replication_factor' : '1', 'class' : 'org.apache.cassandra.locator.SimpleStrategy'}</td>
<td class="col-md-2">false</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#8fb85e21-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="8fb85e21-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE KEYSPACE samples WITH REPLICATION = {
'class' : 'org.apache.cassandra.locator.SimpleStrategy',
'replication_factor': '1' }
AND DURABLE_WRITES = false;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#8fb996a0-3518-11e5-aee3-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;system</span>
</a>
</h4>
</div>
<div id="8fb996a0-3518-11e5-aee3-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-10">Replication</th>
<th class="col-md-2">Durable Writes</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-10">{'class' : 'org.apache.cassandra.locator.LocalStrategy'}</td>
<td class="col-md-2">true</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#8fb9bdb0-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="8fb9bdb0-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE KEYSPACE system WITH REPLICATION = {
'class' : 'org.apache.cassandra.locator.LocalStrategy' }
AND DURABLE_WRITES = true;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#8fb9e4c0-3518-11e5-aee3-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;system_traces</span>
</a>
</h4>
</div>
<div id="8fb9e4c0-3518-11e5-aee3-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-10">Replication</th>
<th class="col-md-2">Durable Writes</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-10">{'replication_factor' : '2', 'class' : 'org.apache.cassandra.locator.SimpleStrategy'}</td>
<td class="col-md-2">true</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#8fb9e4c1-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="8fb9e4c1-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE KEYSPACE system_traces WITH REPLICATION = {
'class' : 'org.apache.cassandra.locator.SimpleStrategy',
'replication_factor': '2' }
AND DURABLE_WRITES = true;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#8fba32e0-3518-11e5-aee3-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;zeppelin</span>
</a>
</h4>
</div>
<div id="8fba32e0-3518-11e5-aee3-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"></div>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<thead>
<tr>
<th class="col-md-10">Replication</th>
<th class="col-md-2">Durable Writes</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-10">{'replication_factor' : '1', 'class' : 'org.apache.cassandra.locator.SimpleStrategy'}</td>
<td class="col-md-2">false</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#8fba32e1-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="8fba32e1-3518-11e5-aee3-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE KEYSPACE zeppelin WITH REPLICATION = {
'class' : 'org.apache.cassandra.locator.SimpleStrategy',
'replication_factor': '1' }
AND DURABLE_WRITES = false;</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>

326
cassandra/src/test/resources/scalate/DescribeTable_live_data_complex_table.html Normal file
View file

@ -0,0 +1,326 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li>
<a><strong>DESCRIBE TABLE live_data.complex_table;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<caption><h4 class="text-primary"><i class="glyphicon glyphicon-th-list"/>&nbsp;complex_table</h4></caption>
<thead>
<tr>
<th class="col-md-4">Column Type</th>
<th class="col-md-4">Column Name</th>
<th class="col-md-4">Data Type</th>
</tr>
</thead>
<tbody>
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
<span></span>
</td>
<td class="col-md-4">pk1</td>
<td class="col-md-4">uuid</td>
</tr>
<tr class="info">
<td class="col-md-4">
<i class="glyphicon glyphicon-fullscreen" title="Partition Key"/>
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>pk2idx</em> <strong></strong>
</td>
<td class="col-md-4">pk2</td>
<td class="col-md-4">int</td>
</tr>
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
<span></span>
</td>
<td class="col-md-4">my_static1</td>
<td class="col-md-4">text</td>
</tr>
<tr class="warning">
<td class="col-md-4">
<i class="glyphicon glyphicon-pushpin" title="Static Column"/>
<span></span>
</td>
<td class="col-md-4">my_static2</td>
<td class="col-md-4">text</td>
</tr>
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes-alt" title="Sort DESC"/>
<span></span>
</td>
<td class="col-md-4">clustering1</td>
<td class="col-md-4">timestamp</td>
</tr>
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes" title="Sort ASC"/>
&nbsp;<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>clustering2idx</em> <strong></strong>
</td>
<td class="col-md-4">clustering2</td>
<td class="col-md-4">int</td>
</tr>
<tr class="success">
<td class="col-md-4">
<i class="glyphicon glyphicon-sort" title="Clustering Column"/>
&nbsp;
<i class="glyphicon glyphicon-sort-by-attributes-alt" title="Sort DESC"/>
<span></span>
</td>
<td class="col-md-4">clustering3</td>
<td class="col-md-4">text</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">entries_indexed_map</td>
<td class="col-md-4">map&lt;int, text&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>idx1</em> <strong></strong>
</td>
<td class="col-md-4">indexed1</td>
<td class="col-md-4">text</td>
</tr>
<tr>
<td class="col-md-4">
<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>idx2</em> <strong></strong>
</td>
<td class="col-md-4">indexed2</td>
<td class="col-md-4">int</td>
</tr>
<tr>
<td class="col-md-4">
<i class="glyphicon glyphicon-info-sign" title="Indexed Column"/>
<em>keys_map_idx</em> <strong></strong>
</td>
<td class="col-md-4">key_indexed_map</td>
<td class="col-md-4">map&lt;int, text&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">my_list</td>
<td class="col-md-4">list&lt;text&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">my_map</td>
<td class="col-md-4">map&lt;int, text&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">my_udt</td>
<td class="col-md-4">frozen&lt;live_data.address&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">my_udt_list</td>
<td class="col-md-4">frozen&lt;list&lt;frozen&lt;live_data.address&gt;&gt;&gt;</td>
</tr>
<tr>
<td class="col-md-4">
<span></span>
</td>
<td class="col-md-4">simple</td>
<td class="col-md-4">double</td>
</tr>
</tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#26dd4330-350d-11e5-9539-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="26dd4330-350d-11e5-9539-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TABLE live_data.complex_table (
pk1 uuid,
pk2 int,
clustering1 timestamp,
clustering2 int,
clustering3 text,
entries_indexed_map map&lt;int, text&gt;,
indexed1 text,
indexed2 int,
key_indexed_map map&lt;int, text&gt;,
my_list list&lt;text&gt;,
my_map map&lt;int, text&gt;,
my_static1 text static,
my_static2 text static,
my_udt frozen&lt;live_data.address&gt;,
my_udt_list frozen&lt;list&lt;frozen&lt;live_data.address&gt;&gt;&gt;,
simple double,
PRIMARY KEY ((pk1, pk2), clustering1, clustering2, clustering3)
) WITH CLUSTERING ORDER BY (clustering1 DESC, clustering2 ASC, clustering3 DESC)
AND read_repair_chance = 0.0
AND dclocal_read_repair_chance = 0.1
AND gc_grace_seconds = 864000
AND bloom_filter_fp_chance = 0.01
AND caching = { 'keys' : 'ALL', 'rows_per_partition' : 'NONE' }
AND comment = ''
AND compaction = { 'class' : 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy' }
AND compression = { 'sstable_compression' : 'org.apache.cassandra.io.compress.LZ4Compressor' }
AND default_time_to_live = 0
AND speculative_retry = '99.0PERCENTILE'
AND min_index_interval = 128
AND max_index_interval = 2048;
CREATE INDEX pk2idx ON live_data.complex_table (pk2);
CREATE INDEX clustering2idx ON live_data.complex_table (clustering2);
CREATE INDEX idx1 ON live_data.complex_table (indexed1);
CREATE INDEX idx2 ON live_data.complex_table (indexed2);
CREATE INDEX keys_map_idx ON live_data.complex_table (KEYS(key_indexed_map));</pre>
</div>
</div>
</div>
</div>
<div class="col-md-2"></div>
</div>

313
cassandra/src/test/resources/scalate/DescribeTables.html Normal file
View file

@ -0,0 +1,313 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li role="presentation" class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-muted"><i class="glyphicon glyphicon-dashboard"/>&nbsp;<strong>Test Cluster</strong></span>
<span class="text-muted caret"></span>
<ul class="dropdown-menu">
<li class="dropdown-header"><span class="text-danger">Keyspaces</span></li>
<li>
<a role="button" data-toggle="collapse" data-target="#d0b77780-3518-11e5-86fc-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;live_data</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#d0b7ecb0-3518-11e5-86fc-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;samples</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#d0b7ecb1-3518-11e5-86fc-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;system</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#d0b813c0-3518-11e5-86fc-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;system_traces</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#d0b813c1-3518-11e5-86fc-8f0ea8ae1a37">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;zeppelin</span>
</a>
</li>
</ul>
</a>
</li>
<li>
<a><strong>DESCRIBE TABLES;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="container">
<div class="row">
<div class="panel-group" role="tablist" aria-multiselectable="true">
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#d0b77780-3518-11e5-86fc-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;live_data</span>
</a>
</h4>
</div>
<div id="d0b77780-3518-11e5-86fc-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<table class="table">
<thead>
<tr><th>Tables</th></tr>
</thead>
<tbody>
<tr class="text-primary"><td>complex_table</td></tr>
<tr class="text-primary"><td>sensor_data</td></tr>
<tr class="text-primary"><td>stations</td></tr>
</tbody>
</table>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#d0b7ecb0-3518-11e5-86fc-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;samples</span>
</a>
</h4>
</div>
<div id="d0b7ecb0-3518-11e5-86fc-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<span><h4>No Table</h4></span>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#d0b7ecb1-3518-11e5-86fc-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;system</span>
</a>
</h4>
</div>
<div id="d0b7ecb1-3518-11e5-86fc-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<table class="table">
<thead>
<tr><th>Tables</th></tr>
</thead>
<tbody>
<tr class="text-primary"><td>IndexInfo</td></tr>
<tr class="text-primary"><td>batchlog</td></tr>
<tr class="text-primary"><td>compaction_history</td></tr>
<tr class="text-primary"><td>compactions_in_progress</td></tr>
<tr class="text-primary"><td>hints</td></tr>
<tr class="text-primary"><td>local</td></tr>
<tr class="text-primary"><td>paxos</td></tr>
<tr class="text-primary"><td>peer_events</td></tr>
<tr class="text-primary"><td>peers</td></tr>
<tr class="text-primary"><td>range_xfers</td></tr>
<tr class="text-primary"><td>schema_columnfamilies</td></tr>
<tr class="text-primary"><td>schema_columns</td></tr>
<tr class="text-primary"><td>schema_keyspaces</td></tr>
<tr class="text-primary"><td>schema_triggers</td></tr>
<tr class="text-primary"><td>schema_usertypes</td></tr>
<tr class="text-primary"><td>size_estimates</td></tr>
<tr class="text-primary"><td>sstable_activity</td></tr>
</tbody>
</table>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#d0b813c0-3518-11e5-86fc-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;system_traces</span>
</a>
</h4>
</div>
<div id="d0b813c0-3518-11e5-86fc-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<table class="table">
<thead>
<tr><th>Tables</th></tr>
</thead>
<tbody>
<tr class="text-primary"><td>events</td></tr>
<tr class="text-primary"><td>sessions</td></tr>
</tbody>
</table>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#d0b813c1-3518-11e5-86fc-8f0ea8ae1a37" aria-expanded="false">
<span class="text-danger"><i class="glyphicon glyphicon-folder-open"/>&nbsp;&nbsp;zeppelin</span>
</a>
</h4>
</div>
<div id="d0b813c1-3518-11e5-86fc-8f0ea8ae1a37" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="row">
<div class="col-md-2"/>
<div class="col-md-8 col-offset-md-2 table-responsive table-bordered">
<table class="table">
<thead>
<tr><th>Tables</th></tr>
</thead>
<tbody>
<tr class="text-primary"><td>artists</td></tr>
<tr class="text-primary"><td>prepared</td></tr>
<tr class="text-primary"><td>ts</td></tr>
<tr class="text-primary"><td>users</td></tr>
</tbody>
</table>
</div>
<div class="col-md-2"/>
</div>
</div>
</div>
</div>
</div>
</div>
</div>

137
cassandra/src/test/resources/scalate/DescribeType_live_data_address.html Normal file
View file

@ -0,0 +1,137 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li>
<a><strong>DESCRIBE TYPE live_data.address;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<caption><h4 class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;address</h4></caption>
<thead>
<tr>
<th class="col-md-6">Column Name</th>
<th class="col-md-6">Data Type</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-6">number</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">street</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">zip</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">city</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">country</td>
<td class="col-md-6">text</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#984da320-350d-11e5-a7a9-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="984da320-350d-11e5-a7a9-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TYPE live_data.address (
number int,
street text,
zip int,
city text,
country text
);</pre>
</div>
</div>
</div>
</div>
<div class="col-md-3"></div>
</div>

137
cassandra/src/test/resources/scalate/DescribeType_live_data_address_within_current_keyspace.html Normal file
View file

@ -0,0 +1,137 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li>
<a><strong>DESCRIBE TYPE address;</strong></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<strong>Legend</strong>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<i class="glyphicon glyphicon-dashboard text-muted" />&nbsp;Cluster
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-folder-open text-danger" />&nbsp;&nbsp;Keyspace
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-copyright-mark text-warning" />&nbsp;&nbsp;UDT
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-th-list text-primary" />&nbsp;&nbsp;Table
</a>
</li>
<li class="bg-info">
<a role="button">
<i class="glyphicon glyphicon-fullscreen" />&nbsp;&nbsp;Partition Key
</a>
</li>
<li class="bg-warning">
<a role="button">
<i class="glyphicon glyphicon-pushpin" />&nbsp;&nbsp;Static Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort" />&nbsp;&nbsp;Clustering Column
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes" />&nbsp;&nbsp;Clustering Order ASC
</a>
</li>
<li class="bg-success">
<a role="button">
<i class="glyphicon glyphicon-sort-by-attributes-alt" />&nbsp;&nbsp;Clustering Order DESC
</a>
</li>
<li>
<a role="button">
<i class="glyphicon glyphicon-info-sign" />&nbsp;&nbsp;Indexed Column
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</ul>
</nav>
<hr/>
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3">
<div class="panel panel-default table-responsive table-bordered">
<table class="table">
<caption><h4 class="text-warning"><i class="glyphicon glyphicon-copyright-mark"/>&nbsp;address</h4></caption>
<thead>
<tr>
<th class="col-md-6">Column Name</th>
<th class="col-md-6">Data Type</th>
</tr>
</thead>
<tbody>
<tr>
<td class="col-md-6">number</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">street</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">zip</td>
<td class="col-md-6">int</td>
</tr>
<tr>
<td class="col-md-6">city</td>
<td class="col-md-6">text</td>
</tr>
<tr>
<td class="col-md-6">country</td>
<td class="col-md-6">text</td>
</tr>
<tbody>
</table>
<div class="panel-footer">
<a data-toggle="collapse" data-target="#984da320-350d-11e5-a7a9-8f0ea8ae1a37_asCQL">
<strong>As CQL statement</strong>
<span class="caret"></span>
</a>
<br/><br/>
<div class="collapse" id="984da320-350d-11e5-a7a9-8f0ea8ae1a37_asCQL">
<pre class="well">CREATE TYPE live_data.address (
number int,
street text,
zip int,
city text,
country text
);</pre>
</div>
</div>
</div>
</div>
<div class="col-md-3"></div>
</div>

870
cassandra/src/test/resources/scalate/Help.html Normal file
View file

@ -0,0 +1,870 @@
<br/>
<br/>
<nav class="navbar navbar-default">
<ul class="nav navbar-nav">
<li role="presentation" class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-info"><i class="glyphicon glyphicon-book"/>&nbsp;<strong>Please select ...</strong></span>
<span class="text-info caret"></span>
<ul class="dropdown-menu">
<li class="dropdown-header"><span class="text-info">Topics</span></li>
<li>
<a role="button" data-toggle="collapse" data-target="#d977b63a-9900-4ecd-a438-70eb490d6a48">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Basic Commands</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#eb22d00b-9be8-478a-b4bf-740f98e1e6ec">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Schema Discovery</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#881a5e0b-5e52-4474-ba80-787dfe0a770d">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Query Parameters</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#2af5a125-8754-40fb-a044-bda10395504f">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Prepared Statements</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#4dfe08e1-7ee0-4222-8be0-3d9d43aab38e">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Dynamic Forms</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#6485679a-ab5b-406b-8281-37f586459754">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Interpreter Configuration</span>
</a>
</li>
<li>
<a role="button" data-toggle="collapse" data-target="#2d060cba-7f9a-40de-8cc3-d82586d4321e">
<span class="text-info"><i class="glyphicon glyphicon-bookmark"/>&nbsp;&nbsp;Misc</span>
</a>
</li>
</ul>
</a>
</li>
<li>
<a><span class="text-info"><strong>CASSANDRA INTERPRETER DOCUMENTATION</strong></span></a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
<span class="text-info"><strong>About ...</strong></span>
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li>
<a role="button">
<span class="text-info">Version <strong>1.0</strong></span>
</a>
</li>
<li>
<a role="button">
<span class="text-info">Java Driver Version <strong>2.1.7.1</strong></span>
</a>
</li>
<li>
<a role="button">
<span class="text-info">Author <strong>@doanduyhai</strong></span>
</a>
</li>
</ul>
</li>
<li>
<a href="#"></a>
</li>
</nav>
<br/><br/>
<div class="container">
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#d977b63a-9900-4ecd-a438-70eb490d6a48" aria-expanded="false">
<span class="text-info"><strong>Basic Commands</strong></span>
</a>
</h4>
</div>
<div id="d977b63a-9900-4ecd-a438-70eb490d6a48" class="panel-collapse collapse in" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<h3>I CQL Statements</h3>
<p>This interpreter is compatible with any CQL statement supported by Cassandra. Ex:
<br/><br/>
<div class="row">
<div class="col-md-6 col-md-offset-3">
<pre>
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
SELECT * FROM users WHERE login='jdoe';
</pre>
</div>
</div>
<br/>
Each statement should be separated by a <strong>semi-colon</strong> (;).
<br/>
<strong>Multi-line</strong> statements as well as multiple statements on the <strong>same line</strong>
are also supported as long as they are separated by a semi-colon. Ex:
<br/>
<br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
USE spark_demo;
SELECT * FROM albums_by_country LIMIT 1; SELECT * FROM countries LIMIT 1;
SELECT *
FROM artists
WHERE login='jlennon';
</pre>
</div>
</div>
<br/>
<strong>Batch</strong> statements are supported and can span multiple lines, as well as
<strong>DDL</strong>(CREATE/ALTER/DROP) statements:
<br/>
<br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
BEGIN BATCH
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
INSERT INTO users_preferences(login,account_type) VALUES('jdoe','BASIC');
APPLY BATCH;
CREATE TABLE IF NOT EXISTS test(
key int PRIMARY KEY,
value text
);
</pre>
</div>
</div>
<br/>
CQL statements are <strong>case-insensitive</strong> (except for column names and values).
This means that the following statements are equivalent and valid:
<br/>
<br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
Insert into users(login,name) vAlues('hsue','Helen SUE');
</pre>
</div>
</div>
<br/>
The complete list of all CQL statements and versions can be found below:
<br/><br/>
<div class="row">
<div class="col-md-6 col-md-offset-3">
<table class="table table-bordered">
<thead>
<tr><th>Cassandra version</th><th>Documentation</th></tr>
</thead>
<tbody>
<tr>
<td><strong>2.2</strong></td>
<td>
<a href="http://docs.datastax.com/en/cql/3.3/cql/cqlIntro.html" target="_blank">
http://docs.datastax.com/en/cql/3.3/cql/cqlIntro.html
</a>
</td>
</tr>
<tr>
<td><strong>2.1 & 2.0</strong></td>
<td>
<a href="http://docs.datastax.com/en/cql/3.1/cql/cql_intro_c.html" target="_blank">
http://docs.datastax.com/en/cql/3.1/cql/cql_intro_c.html
</a>
</td>
</tr>
<tr>
<td><strong>1.2</strong></td>
<td>
<a href="http://docs.datastax.com/en/cql/3.0/cql/aboutCQL.html" target="_blank">
http://docs.datastax.com/en/cql/3.0/cql/aboutCQL.html
</a>
</td>
</tr>
</tbody>
</table>
</div>
</div>
</p>
<h3>II Comments</h3>
<p>
It is possible to add comments between statements. Single line comments start with the
<strong>hash</strong> sign (#). Multi-line comments are enclosed between
<strong>&sol;&ast;&ast;</strong> and <strong>&ast;&ast;&sol;</strong>. Ex:
<br/>
<br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
#First comment
INSERT INTO users(login,name) VALUES('jdoe','John DOE');
/**
Multi line
comments
**/
Insert into users(login,name) vAlues('hsue','Helen SUE');
</pre>
</div>
</div>
<br/>
</p>
<h3>III Syntax Validation</h3>
<p>
The interpreters is shipped with a <em>built-in syntax validator</em>. This validator only
checks for <strong>basic syntax errors</strong>. All CQL-related syntax validation is delegated
directly to <strong>Cassandra</strong>
<br/><br/>
Most of the time, syntax errors are due to missing semi-colons between statements or typo errors.
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#eb22d00b-9be8-478a-b4bf-740f98e1e6ec" aria-expanded="false">
<span class="text-info"><strong>Schema Discovery</strong></span>
</a>
</h4>
</div>
<div id="eb22d00b-9be8-478a-b4bf-740f98e1e6ec" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<h3>I Commands For Discovery</h3>
<p>
To make schema discovery easier and more interactive, the following commands are supported:
<br/><br/>
<table class="table table-bordered">
<thead>
<tr><th>Command</th><th>Description</th></tr>
</thead>
<tbody>
<tr>
<td><strong>DESCRIBE CLUSTER;</strong></td>
<td>Show the current cluster name and its partitioner</td>
</tr>
<tr>
<td><strong>DESCRIBE KEYSPACES;</strong></td>
<td>List all existing keyspaces in the cluster and their configuration
(replication factor, durable write ...)</td>
</tr>
<tr>
<td><strong>DESCRIBE TABLES;</strong></td>
<td>List all existing keyspaces in the cluster and for each, all the tables name</td>
</tr>
<tr>
<td><strong>DESCRIBE KEYSPACE &lt;keyspace name&gt;;</strong></td>
<td>Describe the given keyspace configuration and all its table details (name, columns, ...)</td>
</tr>
<tr>
<td><strong>DESCRIBE TABLE <em>(&lt;keyspace name&gt;).</em>&lt;table name&gt;;</strong></td>
<td>
Describe the given table. If the keyspace is not provided, the current
<strong>logged in</strong> keyspace is used. If there is no logged in keyspace,
the default <em>system</em> keyspace is used. If no table is found, an error message is raised
</td>
</tr>
<tr>
<td><strong>DESCRIBE TYPE <em>(&lt;keyspace name&gt;).</em>&lt;type name&gt;;</strong></td>
<td>
Describe the given type(UDT). If the keyspace is not provided, the current
<strong>logged in</strong> keyspace is used. If there is no logged in keyspace,
the default <em>system</em> keyspace is used. If no type is found, an error message is raised
</td>
</tr>
</tbody>
</table>
<br/>
<div class="alert alert-danger" role="alert">
Please note that each <strong>DESCRIBE</strong> command should be ended by <strong>a semi-colon</strong>.
</div>
</p>
<h3>II Schema Display</h3>
<p>
The schema objects (cluster, keyspace, table &amp; type) are displayed in a tabular format.
There is a <strong>drop-down</strong> menu on the top left corner to expand objects details.
On the top right menu is shown the Icon legend.
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#881a5e0b-5e52-4474-ba80-787dfe0a770d" aria-expanded="false">
<span class="text-info"><strong>Query Parameters</strong></span>
</a>
</h4>
</div>
<div id="881a5e0b-5e52-4474-ba80-787dfe0a770d" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<p>
Sometimes you want to be able to pass runtime query parameters to your statements.
Those parameters are <strong>not</strong> part of the CQL specs and are specific to the interpreter.
Below is the list of all parameters:
<br/><br/>
<table class="table table-bordered">
<caption>
<h4>Query Parameters</h4>
</caption>
<thead>
<tr>
<th>Parameter</th>
<th>Syntax</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>Consistency Level</td>
<td><strong>@consistency=<em>value</em></strong></td>
<td>Apply the given consistency level to all queries in the paragraph</td>
</tr>
<tr>
<td>Serial Consistency Level</td>
<td><strong>@serialConsistency=<em>value</em></strong></td>
<td>Apply the given serial consistency level to all queries in the paragraph</td>
</tr>
<tr>
<td>Timestamp</td>
<td><strong>@timestamp=<em>long value</em></strong></td>
<td>Apply the given timestamp to all queries in the paragraph.<br/>
Please note that timestamp value passed directly in CQL statement will override this value
</td>
</tr>
<tr>
<td>Retry Policy</td>
<td><strong>@retryPolicy=<em>value</em></strong></td>
<td>Apply the given retry policy to all queries in the paragraph</td>
</tr>
<tr>
<td>Fetch Size</td>
<td><strong>@fetchSize=<em>int value</em></strong></td>
<td>Apply the given fetch size to all queries in the paragraph</td>
</tr>
</tbody>
</table>
<br/>
Some parameters only accept restricted values:
<br/><br/>
<table class="table table-bordered">
<caption>
<h4>Allowed Values</h4>
</caption>
<thead>
<tr>
<th>Parameter</th>
<th>Possible Values</th>
</tr>
</thead>
<tbody>
<tr>
<td>Consistency Level</td>
<td><strong>ALL, ANY, ONE, TWO, THREE, QUORUM, LOCAL_ONE, LOCAL_QUORUM, EACH_QUORUM</strong></td>
</tr>
<tr>
<td>Serial Consistency Level</td>
<td><strong>SERIAL, LOCAL_SERIAL</strong></td>
</tr>
<tr>
<td>Timestamp</td>
<td>Any long value</td>
</tr>
<tr>
<td>Retry Policy</td>
<td>
<strong>
DEFAULT, DOWNGRADING_CONSISTENCY, FALLTHROUGH, LOGGING_DEFAULT,
LOGGING_DOWNGRADING, LOGGING_FALLTHROUGH
</strong>
</td>
</tr>
<tr>
<td>Fetch Size</td>
<td>Any integer value</td>
</tr>
</tbody>
</table>
<br/>
<div class="alert alert-danger" role="alert">
Please note that you <strong>should not add semi-colon (;)</strong> at the end of each parameter statement
</div>
Some example:
<br/><br/>
<div class="row">
<div class="col-md-8 col-md-offset-2">
<pre>
CREATE TABLE IF NOT EXISTS spark_demo.ts(
key int PRIMARY KEY,
value text
);
TRUNCATE spark_demo.ts;
# Timestamp in the past
@timestamp=10
# Force timestamp directly in the first insert
INSERT INTO spark_demo.ts(key,value) VALUES(1,'first insert') USING TIMESTAMP 100;
# Select some data to make the clock turn
SELECT * FROM spark_demo.albums LIMIT 100;
# Now insert using the timestamp parameter set at the beginning(10)
INSERT INTO spark_demo.ts(key,value) VALUES(1,'second insert');
# Check for the result. You should see 'first insert'
SELECT value FROM spark_demo.ts WHERE key=1;
</pre>
</div>
</div>
<br/>
Some remarks about query parameters:
<br/><br/>
<div class="alert alert-info" role="alert">
<ul>
<li><strong>many</strong> query parameters can be set in the same paragraph</li>
<li>if the <strong>same</strong> query parameter is set many time with different values,
the interpreter only take into account the first value
</li>
<li>each query parameter applies to <strong>all</strong> CQL statement in the same paragraph,
unless you override the option using plain CQL text (like forcing timestamp with the USING clause)
</li>
<li>the order of each query parameter with regard to CQL statement does not matter</li>
</ul>
</div>
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#2af5a125-8754-40fb-a044-bda10395504f" aria-expanded="false">
<span class="text-info"><strong>Prepared Statements</strong></span>
</a>
</h4>
</div>
<div id="2af5a125-8754-40fb-a044-bda10395504f" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<h3>I Syntax</h3>
<br/>
<p>
For performance reason, it is better to <strong>prepare statements</strong> before-hand and reuse
them later by providing bound values. This interpreter provides 3 commands to handle prepared and
bound statements:
<br/><br/>
<ol>
<li><strong>@prepare</strong></li>
<li><strong>@bind</strong></li>
<li><strong>@remove_prepared</strong></li>
</ol>
<br/>
Example:
<br/>
<div class="row">
<div class="col-md-10 col-md-offset-1">
<pre>
@prepare[statement_name]=...
@bind[statement_name]=’text’, 1223, ’2015-07-30 12:00:01’, null, true, [‘list_item1’, ’list_item2’]
@bind[statement_name_with_no_bound_value]
@remove_prepare[statement_name]
</pre>
</div>
</div>
<br/>
<h3>II @prepare</h3>
<br/>
<p>
You can use the syntax "<strong>@prepare[statement_name]=SELECT ...</strong>" to create a prepared statement.
The <em>statement_name</em> is mandatory because the interpreter prepares the given statement with the
Java driver and saves the generated prepared statement in an internal map, using the provided
<em>statement_name</em> as search key.
<br/><br/>
<div class="alert alert-info">
Please note that this internal prepared statement map is shared with <strong>all notebooks</strong>
and <strong>all paragraphs</strong> because there is only one instance of the interpreter for Cassandra
</div>
<br/>
<div class="alert alert-warning">
If the interpreter encounters many @prepare for the <strong>same statement_name</strong> (key),
only the <strong>first</strong> statement will be taken into account.
</div>
<br/>
Example:
<br/>
<div class="row">
<div class="col-md-10 col-md-offset-1">
<pre>
@prepare[select]=SELECT * FROM spark_demo.albums LIMIT ?
@prepare[select]=SELECT * FROM spark_demo.artists LIMIT ?
</pre>
</div>
</div>
<br/>
For the above example, the prepared statement is <strong>"SELECT * FROM spark_demo.albums LIMIT ?"</strong>.
<em>"SELECT * FROM spark_demo.artists LIMIT ?"</em> is ignored because an entry already exists in the
prepared statements map with the key <strong>select</strong>.
<br/><br/>
In the context of Zeppelin, a notebook can be scheduled to be executed at regular interval,
thus it is necessary to avoid re-preparing many time the same statement (considered an anti-pattern).
</p>
<h3>III @bind</h3>
<br/>
<p>
Once the statement is prepared (possibly in a separated notebook/paragraph). You can bind values to it:
<br/><br/>
<div class="row">
<div class="col-md-10 col-md-offset-1">
<pre>
@bind[select_first]=10
</pre>
</div>
</div>
<br/>
Bound values are not mandatory for the <strong>@bind</strong> statement.
However if you provide bound values, they need to comply to some syntax:
<ul>
<li>String values should be enclosed between simple quotes ( ‘ )</li>
<li>Date values should be enclosed between simple quotes ( ‘ ) and respect the formats:
<ol>
<li>yyyy-MM-dd HH:MM:ss</li>
<li>yyyy-MM-dd HH:MM:ss.SSS</li>
</ol>
</li>
<li><strong>null</strong> is parsed as-is</li>
<li><strong>boolean</strong> (true|false) are parsed as-is </li>
<li>collection values must follow the
<a href="http://docs.datastax.com/en/cql/3.1/cql/cql_using/use_collections_c.html" target="_blank">standard CQL syntax</a>:
<ul>
<li>list: [‘list_item1’, ’list_item2’, ...]</li>
<li>set: {‘set_item1’, ‘set_item2’, …}</li>
<li>map: {‘key1’: ‘val1’, ‘key2’: ‘val2’, …}</li>
</ul>
</li>
<li>
tuple values should be enclosed between parenthesis
(see <a href="http://docs.datastax.com/en/cql/3.1/cql/cql_reference/tupleType.html" target="_blank">tuple CQL syntax</a>):
(‘text’, 123, true)
</li>
<li>
udt values should be enclosed between brackets
(see <a href="http://docs.datastax.com/en/cql/3.1/cql/cql_using/cqlUseUDT.html" target="_blank">udt CQL syntax</a>):
{stree_name: ‘Beverly Hills’, number: 104, zip_code: 90020, state: ‘California’, …}
</li>
</ul>
<br/>
<div class="alert alert-info">
It is possible to use the <strong>@bind</strong> statement inside a batch: <br/>
<pre>
BEGIN BATCH
@bind[insert_user]='jdoe','John DOE'
UPDATE users SET age = 27 WHERE login='hsue';
APPLY BATCH;
</pre>
</div>
<br/>
</p>
<h3>IV @remove_prepare</h3>
<br/>
<p>
To avoid for a prepared statement to stay forever in the prepared statement map, you can use the <strong>@remove_prepare[statement_name]</strong> syntax
to remove it. Removing a non-existing prepared statement yields no error.
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#4dfe08e1-7ee0-4222-8be0-3d9d43aab38e" aria-expanded="false">
<span class="text-info"><strong>Dynamic Forms</strong></span>
</a>
</h4>
</div>
<div id="4dfe08e1-7ee0-4222-8be0-3d9d43aab38e" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<div class="panel panel-default">
<div class="panel-body">
<p>
Instead of hard-coding your CQL queries, it is possible to use the mustache syntax (<strong>{{ }}</strong>)
to inject simple value or multiple choices forms.
<br/><br/>
The syntax for simple parameter is: <strong>{{input_Label=default value}}</strong>.
The default value is mandatory because the first time the paragraph is executed,
we launch the CQL query before rendering the form so at least one value should be provided.
<br/><br/>
The syntax for multiple choices parameter is: <strong>{{input_Label=value1 | value2 | … | valueN }}</strong>.
By default the first choice is used for CQL query the first time the paragraph is executed.
<br/><br/>
Example:
<br/>
<div class="row">
<div class="col-md-10 col-md-offset-1">
<pre>
#Secondary index on performer style
SELECT name, country, performer
FROM spark_demo.performers
WHERE name='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}'
AND styles CONTAINS '{{style=Rock}}';
</pre>
</div>
</div>
<br/>
In the above example, the first CQL query will be executed for <em>performer='Sheryl Crow'</em>
AND <em>style='Rock'</em>. For subsequent queries, you can change the value directly using the form.
Please note that we enclosed the {{ }} block between simple quotes (') because Cassandra expects a String here.
We could have also use the <strong>{{style='Rock'}}</strong> syntax but this time, the value
displayed on the form is <em>'Rock'</em> and not <em>Rock</em>.
<br/><br/>
<div class="alert alert-info">
It is also possible to use dynamic forms for <strong>prepared statements</strong>: <br/>
<strong>@bind[select]=='{{performer=Sheryl Crow|Doof|Fanfarlo|Los Paranoia}}', '{{style=Rock}}'</strong>
</div>
</pre>
</p>
</div>
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#6485679a-ab5b-406b-8281-37f586459754" aria-expanded="false">
<span class="text-info"><strong>Interpreter Configuration</strong></span>
</a>
</h4>
</div>
<div id="6485679a-ab5b-406b-8281-37f586459754" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
The <strong>Cassandra</strong> interpreter comes with some some configuration values for the Java driver:
<table class="table table-bordered">
<caption>
<h4>Interpreter Configuration</h4>
</caption>
<thead>
<tr>
<th>Parameter</th>
<th>Default Value</th>
</tr>
</thead>
<tbody>
<tr>
<td>cassandra.cluster</td>
<td><strong>Test Cluster</strong></td>
</tr>
<tr>
<td>cassandra.compression.protocol</td>
<td><strong>NONE</strong>, possible values: LZ4, SNAPPY</td>
</tr>
<tr>
<td>cassandra.credentials.password</td>
<td><strong>none</strong></td>
</tr>
<tr>
<td>cassandra.credentials.username</td>
<td><strong>none</strong></td>
</tr>
<tr>
<td>cassandra.hosts</td>
<td><strong>localhost</strong></td>
</tr>
<tr>
<td>cassandra.interpreter.parallelism</td>
<td><strong>10</strong></td>
</tr>
<tr>
<td>cassandra.keyspace</td>
<td><strong>system</strong></td>
</tr>
<tr>
<td>cassandra.load.balancing.policy</td>
<td><strong>DEFAULT</strong>, or a FQCN of a custom class</td>
</tr>
<tr>
<td>cassandra.max.schema.agreement.wait.second</td>
<td><strong>10</strong></td>
</tr>
<tr>
<td>cassandra.native.port</td>
<td><strong>9042</strong></td>
</tr>
<tr>
<td>cassandra.pooling.core.connection.per.host.local</td>
<td><strong>Protocol V2 and below: 2, V3 and above: 1</strong></td>
</tr>
<tr>
<td>cassandra.pooling.core.connection.per.host.remote</td>
<td><strong>Protocol V2 and below: 1, V3 and above: 1</strong></td>
</tr>
<tr>
<td>cassandra.pooling.heartbeat.interval.seconds</td>
<td><strong>30</strong></td>
</tr>
<tr>
<td>cassandra.pooling.idle.timeout.seconds</td>
<td><strong>Test Cluster</strong></td>
</tr>
<tr>
<td>cassandra.pooling.max.connection.per.host.local</td>
<td><strong>Protocol V2 and below: 8, V3 and above: 1</strong></td>
</tr>
<tr>
<td>cassandra.pooling.max.connection.per.host.remote</td>
<td><strong>Protocol V2 and below: 2, V3 and above: 1</strong></td>
</tr>
<tr>
<td>cassandra.pooling.max.request.per.connection.local</td>
<td><strong>Protocol V2 and below: 128, V3 and above: 1024</strong></td>
</tr>
<tr>
<td>cassandra.pooling.max.request.per.connection.remote</td>
<td><strong>Protocol V2 and below: 128, V3 and above: 256</strong></td>
</tr>
<tr>
<td>cassandra.pooling.new.connection.threshold.local</td>
<td><strong>Protocol V2 and below: 100, V3 and above: 800</strong></td>
</tr>
<tr>
<td>cassandra.pooling.new.connection.threshold.remote</td>
<td><strong>Protocol V2 and below: 100, V3 and above: 200</strong></td>
</tr>
<tr>
<td>cassandra.pooling.pool.timeout.millisecs</td>
<td><strong>5000</strong></td>
</tr>
<tr>
<td>cassandra.protocol.version</td>
<td><strong>3</strong></td>
</tr>
<tr>
<td>cassandra.query.default.consistency</td>
<td><strong>ONE</strong></td>
</tr>
<tr>
<td>cassandra.query.default.fetchSize</td>
<td><strong>5000</strong></td>
</tr>
<tr>
<td>cassandra.query.default.serial.consistency</td>
<td><strong>SERIAL</strong></td>
</tr>
<tr>
<td>cassandra.reconnection.policy</td>
<td><strong>DEFAULT</strong>, or a FQCN of a custom class</td>
</tr>
<tr>
<td>cassandra.retry.policy</td>
<td><strong>DEFAULT</strong>, or a FQCN of a custom class</td>
</tr>
<tr>
<td>cassandra.socket.connection.timeout.millisecs</td>
<td><strong>500</strong></td>
</tr>
<tr>
<td>cassandra.socket.read.timeout.millisecs</td>
<td><strong>12000</strong></td>
</tr>
<tr>
<td>cassandra.socket.tcp.no_delay</td>
<td><strong>true</strong></td>
</tr>
<tr>
<td>cassandra.speculative.execution.policy</td>
<td><strong>DEFAULT</strong>, or a FQCN of a custom class</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading" role="tab">
<h4 class="panel-title">
<a role="button" data-toggle="collapse" data-target="#2d060cba-7f9a-40de-8cc3-d82586d4321e" aria-expanded="false">
<span class="text-info"><strong>Miscellaneous</strong></span>
</a>
</h4>
</div>
<div id="2d060cba-7f9a-40de-8cc3-d82586d4321e" class="panel-collapse collapse" role="tabpanel">
<div class="panel-body">
<h3>Execution parallelism</h3>
It is possible to execute many paragraphs in parallel. However, at the back-end side, we’re still using <strong>synchronous</strong> queries. Asynchronous execution is only possible when it is possible to return a Future value in the <strong>InterpreterResult</strong>. It may be an interesting proposal for the Zeppelin project.
</div>
</div>
</div>
</div>

6
cassandra/src/test/resources/scalate/NoResult.html Normal file
View file

@ -0,0 +1,6 @@
<div class="container">
<div class="row text-center">
<h4>No Result</h4>
</div>
<br>
</div>

42
cassandra/src/test/resources/scalate/NoResultWithExecutionInfo.html Normal file
View file

@ -0,0 +1,42 @@
<div class="container">
<div class="row text-center">
<h4>No Result</h4>
</div>
<br/>
<div class="row">
<div class="col-md-3"></div>
<div class="col-md-6 col-offset-md-3 table-responsive table-bordered">
<table class="table">
<caption><h5>Last query execution info</h5></caption>
<thead>
<tr>
<th>Info</th>
<th>Value</th>
</tr>
</thead>
<tbody>
<tr>
<td>Statement</td>
<td>CREATE TABLE IF NOT EXISTS no_select(id int PRIMARY KEY);</td>
</tr>
<tr>
<td>Achieved Consistency</td>
<td>N/A</td>
</tr>
<tr>
<td>Tried Hosts</td>
<td>TRIED_HOSTS</td>
</tr>
<tr>
<td>Queried Hosts</td>
<td>QUERIED_HOSTS</td>
</tr>
<tr>
<td>Schema In Agreement</td>
<td>true</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>

193
cassandra/src/test/scala/org/apache/zeppelin/cassandra/BoundValuesParserTest.scala Normal file
View file

@ -0,0 +1,193 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import org.scalatest.{Matchers, BeforeAndAfterEach, FlatSpec}
class BoundValuesParserTest extends FlatSpec
with BeforeAndAfterEach
with Matchers {
val parser = new BoundValuesParser
"BoundValuesParser" should "parse quoted string" in {
//Given
val input = """'a'"""
//When
val parsed1 = parser.parse(parser.quotedString,input)
val parsed2 = parser.parse(parser.value,input)
//Then
parsed1.get should be("'a'")
parsed2.get should be("'a'")
}
"BoundValuesParser" should "parse integer" in {
//Given
val input = """00123"""
//When
val parsed = parser.parse(parser.value,input)
//Then
parsed.get should be("123")
}
"BoundValuesParser" should "parse decimal number" in {
//Given
val input1 = """00123.35000"""
val input2 = """+123."""
val input3 = """.35000"""
val input4 = """-.35000"""
//When
val parsed1 = parser.parse(parser.value,input1)
val parsed2 = parser.parse(parser.value,input2)
val parsed3 = parser.parse(parser.value,input3)
val parsed4 = parser.parse(parser.value,input4)
//Then
parsed1.get should be("123.35")
parsed2.get should be("123.0")
parsed3.get should be("0.35")
parsed4.get should be("-0.35")
}
"BoundValuesParser" should "parse list" in {
//Given
val input = """['a','b','c']"""
//When
val parsed = parser.parse(parser.value,input)
//Then
parsed.get should be("['a','b','c']")
}
"BoundValuesParser" should "parse set" in {
//Given
val input = """{'a',2,3.4}"""
//When
val parsed = parser.parse(parser.value,input)
//Then
parsed.get should be("{'a',2,3.4}")
}
"BoundValuesParser" should "parse map" in {
//Given
val input = """{'key1': 'val', 'key2': 2, 'key3': 3.4}"""
//When
val parsed = parser.parse(parser.value,input)
//Then
parsed.get should be("{'key1': 'val', 'key2': 2, 'key3': 3.4}")
}
"BoundValuesParser" should "parse tuple" in {
//Given
val input = """('a',2,3.4)"""
//When
val parsed = parser.parse(parser.value,input)
//Then
parsed.get should be("('a',2,3.4)")
}
"BoundValuesParser" should "parse udt" in {
//Given
val input = """{col1: 'val1', col2: 2, col3: 3.4}"""
//When
val parsed = parser.parse(parser.value,input)
//Then
parsed.get should be("{col1: 'val1', col2: 2, col3: 3.4}")
}
"BoundValuesParser" should "parse date" in {
//Given
val input = """'2015-07-10 14:56:34'"""
//When
val parsed = parser.parse(parser.value,input)
//Then
parsed.get should be("2015-07-10 14:56:34")
}
"BoundValuesParser" should "parse nested types" in {
//Given
val input = "'jdoe','John','DOE'," +
"{street_number: 3, street_name: 'Beverly Hills Bld', zip_code: 90209," +
" country: 'USA', extra_info: ['Right on the hills','Next to the post box']," +
" phone_numbers: {'home': 2016778524, 'office': 2015790847} }," +
"('USA', 90209, 'Beverly Hills')"
//When
val parsed = parser.parse(parser.values,input)
//Then
parsed.get should be(List(
"'jdoe'",
"'John'",
"'DOE'",
"{street_number: 3, street_name: 'Beverly Hills Bld', zip_code: 90209, country: 'USA', extra_info: ['Right on the hills','Next to the post box'], phone_numbers: {'home': 2016778524, 'office': 2015790847}}",
"('USA',90209,'Beverly Hills')"
))
}
"BoundValuesParser" should "not parse mustaches for zeppelin variables" in {
//Given
val input = "'jdoe',{{firstname='Jdoe'}}"
//When
val parsed = parser.parse(parser.values,input)
//Then
parsed.get should be(List("'jdoe'","{{firstname='Jdoe'}}"))
}
"BoundValuesParser" should "parse zeppelin variable" in {
//Given
val input = """'{{login=jdoe}}'"""
//When
val parsed = parser.parse(parser.values,input)
//Then
parsed.get should be(List("'{{login=jdoe}}'"))
}
"BoundValuesParser" should "parse null value" in {
//Given
val input = """'login',null,'LASTNAME'"""
//When
val parsed = parser.parse(parser.values,input)
//Then
parsed.get should be(List("'login'","null","'LASTNAME'"))
}
}

585
cassandra/src/test/scala/org/apache/zeppelin/cassandra/ParagraphParserTest.scala Normal file
View file

@ -0,0 +1,585 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.cassandra
import com.datastax.driver.core._
import org.apache.zeppelin.interpreter.InterpreterException
import org.scalatest.mock.MockitoSugar
import org.scalatest.{BeforeAndAfterEach, FlatSpec, Matchers}
import org.apache.zeppelin.cassandra.ParagraphParser._
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
class ParagraphParserTest extends FlatSpec
with BeforeAndAfterEach
with Matchers
with MockitoSugar {
val session: Session = mock[Session]
val preparedStatements:collection.mutable.Map[String,PreparedStatement] = collection.mutable.Map()
val parser: ParagraphParser = new ParagraphParser()
"Parser" should "parse mixed statements" in {
val query: String = """
SELECT * FROM albums LIMIT 10;
begin UnLoGgEd BATCH
INSERT INTO users(id) VALUES(10);
@bind[test]='a',12.34
apply Batch;
SELECT * FROM users LIMIT 10;
BEGIN BATCH
Insert INTO users(id) VALUES(11);
INSERT INTO users(id) VALUES(12);
APPLY BATCH;
@bind[toto]='a',12.34
desc table zeppelin.users;
describe keyspace zeppelin;
""".stripMargin
val parsed = parser.parse(parser.queries,query)
parsed.get should be(List(
SimpleStm("SELECT * FROM albums LIMIT 10;"),
BatchStm(BatchStatement.Type.UNLOGGED,
List(
SimpleStm("INSERT INTO users(id) VALUES(10);"),
BoundStm("test","'a',12.34")
)
),
SimpleStm("SELECT * FROM users LIMIT 10;"),
BatchStm(BatchStatement.Type.LOGGED,
List(
SimpleStm("Insert INTO users(id) VALUES(11);"),
SimpleStm("INSERT INTO users(id) VALUES(12);")
)
),
BoundStm("toto","'a',12.34"),
DescribeTableCmd(Option("zeppelin"),"users"),
DescribeKeyspaceCmd("zeppelin")
))
}
"Parser" should "parse single-line comment" in {
val query :CharSequence="""#This is a comment""".stripMargin
val parsed = parser.parseAll[Comment](parser.singleLineComment, query)
parsed.get should be(Comment("This is a comment"))
}
"Parser" should "parse multi-line comment" in {
val query:String =
"""/*This is a comment
|line1
|line2
|line3
|*/
""".stripMargin
val parsed = parser.parseAll(parser.multiLineComment, query)
parsed.get should be(Comment("This is a comment\nline1\nline2\nline3\n"))
}
"Parser" should "parse consistency level" in {
val query:String =""" @consistency=ONE""".stripMargin
val parsed = parser.parseAll(parser.consistency, query)
parsed.get should be(Consistency(ConsistencyLevel.ONE))
}
"Parser" should "fails parsing unknown consistency level" in {
val query:String =""" @consistency=TEST""".stripMargin
val ex = intercept[InterpreterException] {
parser.parseAll(parser.consistency, query)
}
ex.getMessage should be(s"Invalid syntax for @consistency. It should comply to the pattern ${CONSISTENCY_LEVEL_PATTERN.toString}")
}
"Parser" should "parse serial consistency level" in {
val query:String =""" @serialConsistency=LOCAL_SERIAL""".stripMargin
val parsed = parser.parseAll(parser.serialConsistency, query)
parsed.get should be(SerialConsistency(ConsistencyLevel.LOCAL_SERIAL))
}
"Parser" should "fails parsing unknown serial consistency level" in {
val query:String =""" @serialConsistency=TEST""".stripMargin
val ex = intercept[InterpreterException] {
parser.parseAll(parser.serialConsistency, query)
}
ex.getMessage should be(s"Invalid syntax for @serialConsistency. It should comply to the pattern ${SERIAL_CONSISTENCY_LEVEL_PATTERN.toString}")
}
"Parser" should "parse timestamp" in {
val query:String =""" @timestamp=111""".stripMargin
val parsed = parser.parseAll(parser.timestamp, query)
parsed.get should be(Timestamp(111L))
}
"Parser" should "fails parsing invalid timestamp" in {
val query:String =""" @timestamp=TEST""".stripMargin
val ex = intercept[InterpreterException] {
parser.parseAll(parser.timestamp, query)
}
ex.getMessage should be(s"Invalid syntax for @timestamp. It should comply to the pattern ${TIMESTAMP_PATTERN.toString}")
}
"Parser" should "parse retry policy" in {
val query:String ="@retryPolicy="+CassandraInterpreter.DOWNGRADING_CONSISTENCY_RETRY
val parsed = parser.parseAll(parser.retryPolicy, query)
parsed.get should be(DowngradingRetryPolicy)
}
"Parser" should "fails parsing invalid retry policy" in {
val query:String =""" @retryPolicy=TEST""".stripMargin
val ex = intercept[InterpreterException] {
parser.parseAll(parser.retryPolicy, query)
}
ex.getMessage should be(s"Invalid syntax for @retryPolicy. It should comply to the pattern ${RETRY_POLICIES_PATTERN.toString}")
}
"Parser" should "parse fetch size" in {
val query:String ="@fetchSize=100"
val parsed = parser.parseAll(parser.fetchSize, query)
parsed.get should be(FetchSize(100))
}
"Parser" should "fails parsing invalid fetch size" in {
val query:String =""" @fetchSize=TEST""".stripMargin
val ex = intercept[InterpreterException] {
parser.parseAll(parser.fetchSize, query)
}
ex.getMessage should be(s"Invalid syntax for @fetchSize. It should comply to the pattern ${FETCHSIZE_PATTERN.toString}")
}
"Parser" should "parse simple statement" in {
//Given
val query:String =""" sElecT * FROM users LIMIT ? ;""".stripMargin
//When
val parsed = parser.parseAll(parser.genericStatement, query)
//Then
parsed.get should be(SimpleStm("sElecT * FROM users LIMIT ? ;"))
}
"Parser" should "parse prepare" in {
//Given
val query:String =""" @prepare[select_users]=SELECT * FROM users LIMIT ? """.stripMargin
//When
val parsed = parser.parseAll(parser.prepare, query)
//Then
parsed.get should be(PrepareStm("select_users","SELECT * FROM users LIMIT ?"))
}
"Parser" should "fails parsing invalid prepared statement" in {
val query:String =""" @prepare=SELECT * FROM users LIMIT ?""".stripMargin
val ex = intercept[InterpreterException] {
parser.parseAll(parser.prepare, query)
}
ex.getMessage should be(s"Invalid syntax for @prepare. It should comply to the pattern: @prepare[prepared_statement_name]=CQL Statement (without semi-colon)")
}
"Parser" should "parse remove prepare" in {
//Given
val query:String =""" @remove_prepare[select_users ]""".stripMargin
//When
val parsed = parser.parseAll(parser.removePrepare, query)
//Then
parsed.get should be(RemovePrepareStm("select_users"))
}
"Parser" should "fails parsing invalid remove prepared statement" in {
val query:String =""" @remove_prepare[select_users]=SELECT * FROM users LIMIT ?""".stripMargin
val ex = intercept[InterpreterException] {
parser.parseAll(parser.removePrepare, query)
}
ex.getMessage should be(s"Invalid syntax for @remove_prepare. It should comply to the pattern: @remove_prepare[prepared_statement_name]")
}
"Parser" should "parse bind" in {
//Given
val query:String =""" @bind[select_users ]=10,'toto'""".stripMargin
//When
val parsed = parser.parseAll(parser.bind, query)
//Then
parsed.get should be(BoundStm("select_users","10,'toto'"))
}
"Parser" should "fails parsing invalid bind statement" in {
val query:String =""" @bind[select_users]=""".stripMargin
val ex = intercept[InterpreterException] {
parser.parseAll(parser.bind, query)
}
ex.getMessage should be("""Invalid syntax for @bind. It should comply to the pattern: @bind[prepared_statement_name]=10,'jdoe','John DOE',12345,'2015-07-32 12:04:23.234' OR @bind[prepared_statement_name] with no bound value. No semi-colon""")
}
"Parser" should "parse batch" in {
//Given
val query:String ="""
bEgin Batch
Insert INTO users(id) VALUES(10);
@bind[select_users ]=10,'toto'
update users SET name ='John DOE' WHERE id=10;
dElEtE users WHERE id=11;
APPLY BATCH;""".stripMargin
//When
val parsed = parser.parseAll(parser.batch, query)
//Then
parsed.get should be(
BatchStm(
BatchStatement.Type.LOGGED,
List[QueryStatement](
SimpleStm("Insert INTO users(id) VALUES(10);"),
BoundStm("select_users", "10,'toto'"),
SimpleStm("update users SET name ='John DOE' WHERE id=10;"),
SimpleStm("dElEtE users WHERE id=11;")
)
)
)
}
"Parser" should "fails parsing invalid batch type" in {
val query:String ="""BEGIN UNKNOWN BATCH""".stripMargin
val ex = intercept[InterpreterException] {
parser.extractBatchType(query)
}
ex.getMessage should be(s"""Invalid syntax for BEGIN BATCH. It should comply to the pattern: ${BATCH_PATTERN.toString}""")
}
"Parser" should "parse query parameter with statement" in {
val query:String = "@serialConsistency=SERIAL\n" +
"SELECT * FROM zeppelin.artists LIMIT 1;"
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
SerialConsistency(ConsistencyLevel.SERIAL),
SimpleStm("SELECT * FROM zeppelin.artists LIMIT 1;")
))
}
"Parser" should "parse multi-line single statement" in {
val query:String = "CREATE TABLE IF NOT EXISTS zeppelin.albums(\n" +
" title text PRIMARY KEY,\n" +
" artist text,\n" +
" year int\n" +
");\n";
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
SimpleStm("CREATE TABLE IF NOT EXISTS zeppelin.albums(\n title text PRIMARY KEY,\n artist text,\n year int\n);")
))
}
"Parser" should "parse multi-line statements" in {
val query:String = "CREATE TABLE IF NOT EXISTS zeppelin.albums(\n" +
" title text PRIMARY KEY,\n" +
" artist text,\n" +
" year int\n" +
");\n" +
"@consistency=THREE\n" +
"@serialConsistency=SERIAL\n" +
"BEGIN BATCH\n"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Impossible Dream EP','Carter the Unstoppable Sex Machine',1992);"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Way You Are','Tears for Fears',1983);"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('Primitive','Soulfly',2003);\n"+
"APPLY BATCH;\n"+
"@timestamp=10\n" +
"@retryPolicy=DOWNGRADING_CONSISTENCY\n" +
"SELECT * FROM zeppelin.albums;";
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
SimpleStm("CREATE TABLE IF NOT EXISTS zeppelin.albums(\n title text PRIMARY KEY,\n artist text,\n year int\n);"),
Consistency(ConsistencyLevel.THREE),
SerialConsistency(ConsistencyLevel.SERIAL),
BatchStm(BatchStatement.Type.LOGGED,
List(
SimpleStm("INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Impossible Dream EP','Carter the Unstoppable Sex Machine',1992);"),
SimpleStm("INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Way You Are','Tears for Fears',1983);"),
SimpleStm("INSERT INTO zeppelin.albums(title,artist,year) VALUES('Primitive','Soulfly',2003);")
)
),
Timestamp(10L),
DowngradingRetryPolicy,
SimpleStm("SELECT * FROM zeppelin.albums;")
))
}
"Parser" should "parse mixed single-line and multi-line statements" in {
val query:String = "CREATE TABLE IF NOT EXISTS zeppelin.albums(\n" +
" title text PRIMARY KEY,\n" +
" artist text,\n" +
" year int\n" +
");\n" +
"BEGIN BATCH"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Impossible Dream EP','Carter the Unstoppable Sex Machine',1992);"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Way You Are','Tears for Fears',1983);"+
" INSERT INTO zeppelin.albums(title,artist,year) VALUES('Primitive','Soulfly',2003);\n"+
"APPLY BATCH;"+
"SELECT * FROM zeppelin.albums;";
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
SimpleStm("CREATE TABLE IF NOT EXISTS zeppelin.albums(\n title text PRIMARY KEY,\n artist text,\n year int\n);"),
BatchStm(BatchStatement.Type.LOGGED,
List(
SimpleStm("INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Impossible Dream EP','Carter the Unstoppable Sex Machine',1992);"),
SimpleStm("INSERT INTO zeppelin.albums(title,artist,year) VALUES('The Way You Are','Tears for Fears',1983);"),
SimpleStm("INSERT INTO zeppelin.albums(title,artist,year) VALUES('Primitive','Soulfly',2003);")
)
),
SimpleStm("SELECT * FROM zeppelin.albums;")
))
}
"Parser" should "parse a block queries with comments" in {
val query =
"""
/*
This example show how to force a
timestamp on the query
*/
#Timestamp in the past
@timestamp=10
CREATE TABLE IF NOT EXISTS spark_demo.ts(key int PRIMARY KEY, value text);
TRUNCATE spark_demo.ts;
#Force timestamp directly in the first INSERT
INSERT INTO spark_demo.ts(key,value) VALUES(1,'val1') USING TIMESTAMP 100;
#Select some data to loose some time
SELECT * FROM spark_demo.albums LIMIT 100;
#Use @timestamp value set at the beginning(10)
INSERT INTO spark_demo.ts(key,value) VALUES(1,'val2');
#Check the result
SELECT * FROM spark_demo.ts WHERE key=1;
""".stripMargin
val parsed = parser.parseAll(parser.queries, query)
parsed.get should be (List(
Comment("\n This example show how to force a\n timestamp on the query\n "),
Comment("Timestamp in the past"),
Timestamp(10L),
SimpleStm("CREATE TABLE IF NOT EXISTS spark_demo.ts(key int PRIMARY KEY, value text);"),
SimpleStm("TRUNCATE spark_demo.ts;"),
Comment("Force timestamp directly in the first INSERT"),
SimpleStm("INSERT INTO spark_demo.ts(key,value) VALUES(1,'val1') USING TIMESTAMP 100;"),
Comment("Select some data to loose some time"),
SimpleStm("SELECT * FROM spark_demo.albums LIMIT 100;"),
Comment("Use @timestamp value set at the beginning(10)"),
SimpleStm("INSERT INTO spark_demo.ts(key,value) VALUES(1,'val2');"),
Comment("Check the result"),
SimpleStm("SELECT * FROM spark_demo.ts WHERE key=1;")
)
)
}
"Parser" should "remove prepared statement" in {
val queries =
"""
#Removing an unknown statement should has no side effect
@remove_prepare[unknown_statement]
@remove_prepare[select_artist_by_name]
#This should fail because the 'select_artist_by_name' has been removed
@bind[select_artist_by_name]='The Beatles'
""".stripMargin
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(
Comment("Removing an unknown statement should has no side effect"),
RemovePrepareStm("unknown_statement"),
RemovePrepareStm("select_artist_by_name"),
Comment("This should fail because the 'select_artist_by_name' has been removed"),
BoundStm("select_artist_by_name","'The Beatles'")
))
}
"Parser" should "parse only parameter" in {
val queries =
"@fetchSize=1000"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(FetchSize(1000)))
}
"Parser" should "parse describe cluster" in {
val queries ="Describe ClUsTeR;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get(0) shouldBe a [DescribeClusterCmd]
}
"Parser" should "fail parsing describe cluster" in {
val queries ="Describe ClUsTeR"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE CLUSTER. It should comply to the pattern: ${DESCRIBE_CLUSTER_PATTERN.toString}")
}
"Parser" should "parse describe keyspaces" in {
val queries ="Describe KeYsPaCeS;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get(0) shouldBe a [DescribeKeyspacesCmd]
}
"Parser" should "fail parsing describe keyspaces" in {
val queries ="Describe KeYsPaCeS"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE KEYSPACES. It should comply to the pattern: ${DESCRIBE_KEYSPACES_PATTERN.toString}")
}
"Parser" should "parse describe tables" in {
val queries ="Describe TaBlEs;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get(0) shouldBe a [DescribeTablesCmd]
}
"Parser" should "fail parsing describe tables" in {
val queries ="Describe TaBlEs"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE TABLES. It should comply to the pattern: ${DESCRIBE_TABLES_PATTERN.toString}")
}
"Parser" should "parse describe keyspace" in {
val queries ="Describe KeYsPaCe toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(DescribeKeyspaceCmd("toto")))
}
"Parser" should "fail parsing describe keyspace" in {
val queries ="Describe KeYsPaCe toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE KEYSPACE. It should comply to the pattern: ${DESCRIBE_KEYSPACE_PATTERN.toString}")
}
"Parser" should "parse describe table" in {
val queries ="Describe TaBlE toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(DescribeTableCmd(None,"toto")))
}
"Parser" should "parse describe table with keyspace" in {
val queries ="Describe TaBlE ks.toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(DescribeTableCmd(Some("ks"),"toto")))
}
"Parser" should "fail parsing describe table" in {
val queries ="Describe TaBlE toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE TABLE. It should comply to the patterns: " +
s"${DESCRIBE_TABLE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TABLE_PATTERN.toString}")
}
"Parser" should "parse describe type" in {
val queries ="Describe Type toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(DescribeUDTCmd(None,"toto")))
}
"Parser" should "parse describe type with keyspace" in {
val queries ="Describe Type ks.toto;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get should be(List(DescribeUDTCmd(Some("ks"),"toto")))
}
"Parser" should "fail parsing describe type" in {
val queries ="Describe Type toto"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for DESCRIBE TYPE. It should comply to the patterns: " +
s"${DESCRIBE_TYPE_WITH_KEYSPACE_PATTERN.toString} or ${DESCRIBE_TYPE_PATTERN.toString}")
}
"Parser" should "parse help" in {
val queries ="hElp;"
val parsed = parser.parseAll(parser.queries, queries)
parsed.get(0) shouldBe a [HelpCmd]
}
"Parser" should "fail parsing help" in {
val queries ="HELP"
val ex = intercept[InterpreterException] {
parser.parseAll(parser.queries, queries)
}
ex.getMessage should be(s"Invalid syntax for HELP. It should comply to the patterns: " +
s"${HELP_PATTERN.toString}")
}
}

1
conf/log4j.properties
View file

@ -21,6 +21,7 @@ log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n
log4j.appender.dailyfile.DatePattern=.yyyy-MM-dd
log4j.appender.dailyfile.Threshold = INFO
log4j.appender.dailyfile = org.apache.log4j.DailyRollingFileAppender
log4j.appender.dailyfile.File = ${zeppelin.log.file}

34
conf/zeppelin-env.sh.template
View file

@ -17,22 +17,30 @@
#
# export JAVA_HOME=
# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode
# export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Defualt = ZEPPELIN_MEM
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
# export MASTER= # Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode.
# export ZEPPELIN_JAVA_OPTS # Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16"
# export ZEPPELIN_MEM # Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m
# export ZEPPELIN_INTP_MEM # zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM
# export ZEPPELIN_INTP_JAVA_OPTS # zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS
# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default.
# export ZEPPELIN_PID_DIR # The pid files are stored. /tmp by default.
# export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_LOG_DIR # Where log files are stored. PWD by default.
# export ZEPPELIN_PID_DIR # The pid files are stored. /tmp by default.
# export ZEPPELIN_NOTEBOOK_DIR # Where notebook saved
# export ZEPPELIN_NOTEBOOK_HOMESCREEN # Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
# export ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE # hide homescreen notebook from list when this value set to "true". default "false"
# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default.
# export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default.
# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default.
# Options read in YARN client mode
# export HADOOP_CONF_DIR # yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR.
# export HADOOP_CONF_DIR # yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR.
# Pyspark (supported with Spark 1.2.1 and above)
# To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI
# export PYSPARK_PYTHON # path to the python command. must be the same path on the driver(Zeppelin) and all workers.
# export PYTHONPATH # extra PYTHONPATH.
# export PYSPARK_PYTHON # path to the python command. must be the same path on the driver(Zeppelin) and all workers.
# export PYTHONPATH # extra PYTHONPATH.

74
conf/zeppelin-site.xml.template Normal file → Executable file
View file

@ -28,22 +28,7 @@
<property>
<name>zeppelin.server.port</name>
<value>8080</value>
<description>Server port. port+1 is used for web socket.</description>
</property>
<property>
<name>zeppelin.websocket.addr</name>
<value>0.0.0.0</value>
<description>Testing websocket address</description>
</property>
<!-- If the port value is negative, then it'll default to the server
port + 1.
-->
<property>
<name>zeppelin.websocket.port</name>
<value>-1</value>
<description>Testing websocket port</description>
<description>Server port.</description>
</property>
<property>
@ -52,6 +37,40 @@
<description>path or URI for notebook persist</description>
</property>
<property>
<name>zeppelin.notebook.homescreen</name>
<value></value>
<description>id of notebook to be displayed in homescreen. ex) 2A94M5J1Z Empty value displays default home screen</description>
</property>
<property>
<name>zeppelin.notebook.homescreen.hide</name>
<value>false</value>
<description>hide homescreen notebook from list when this value set to true</description>
</property>
<!-- If used S3 to storage the notebooks, it is necessary the following folder structure bucketname/username/notebook/ -->
<!--
<property>
<name>zeppelin.notebook.s3.user</name>
<value>user</value>
<description>user name for s3 folder structure</description>
</property>
<property>
<name>zeppelin.notebook.s3.bucket</name>
<value>zeppelin</value>
<description>bucket name for notebook storage</description>
</property>
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.S3NotebookRepo</value>
<description>notebook persistence layer implementation</description>
</property>
-->
<property>
<name>zeppelin.notebook.storage</name>
<value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
@ -66,10 +85,17 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter, org.apache.zeppelin.phoenix.PhoenixInterpreter,org.apache.zeppelin.kylin.KylinInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>
<property>
<name>zeppelin.interpreter.connect.timeout</name>
<value>30000</value>
<description>Interpreter process connect timeout in msec.</description>
</property>
<property>
<name>zeppelin.ssl</name>
<value>false</value>
@ -84,8 +110,8 @@
<property>
<name>zeppelin.ssl.keystore.path</name>
<value>conf/keystore</value>
<description>Path to keystore relative to Zeppelin home</description>
<value>keystore</value>
<description>Path to keystore relative to Zeppelin configuration directory</description>
</property>
<property>
@ -110,8 +136,8 @@
<property>
<name>zeppelin.ssl.truststore.path</name>
<value>conf/truststore</value>
<description>Path to truststore relative to Zeppelin home. Defaults to the keystore path</description>
<value>truststore</value>
<description>Path to truststore relative to Zeppelin configuration directory. Defaults to the keystore path</description>
</property>
<property>
@ -128,5 +154,11 @@
</property>
-->
<property>
<name>zeppelin.server.allowed.origins</name>
<value>*</value>
<description>Allowed sources for REST and WebSocket requests (i.e. http://onehost:8080,http://otherhost.com). If you leave * you are vulnerable to https://issues.apache.org/jira/browse/ZEPPELIN-173</description>
</property>
</configuration>

132
dev/create_release.sh Executable file
View file

@ -0,0 +1,132 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# The script helps making a release.
# You need specify a release name and branch|tag name.
#
# Here's some helpful documents for the release
# http://www.apache.org/dev/release.html
# http://www.apache.org/dev/release-publishing
# http://incubator.apache.org/guides/releasemanagement.html
# http://incubator.apache.org/guides/release.html
# http://www.apache.org/dev/release-signing.html
# http://www.apache.org/dev/publishing-maven-artifacts.html
if [[ -z "${TAR}" ]]; then
TAR=/usr/bin/tar
fi
if [[ -z "${SHASUM}" ]]; then
SHASUM="/usr/bin/shasum -a 512"
fi
if [[ -z "${WORKING_DIR}" ]]; then
WORKING_DIR=/tmp/incubator-zeppelin-release
fi
if [[ -z "${GPG_PASSPHRASE}" ]]; then
echo "You need GPG_PASSPHRASE variable set"
exit 1
fi
if [[ $# -ne 2 ]]; then
echo "usage) $0 [Release name] [Branch or Tag]"
echo " ex. $0 0.5.0-incubating branch-0.5"
exit 1
fi
RELEASE_NAME="${1}"
BRANCH="${2}"
if [[ -d "${WORKING_DIR}" ]]; then
echo "Dir ${WORKING_DIR} already exists"
exit 1
fi
mkdir ${WORKING_DIR}
echo "Cloning the source and packaging"
# clone source
git clone -b ${BRANCH} git@github.com:apache/incubator-zeppelin.git ${WORKING_DIR}/zeppelin
if [[ $? -ne 0 ]]; then
echo "Can not clone source repository"
exit 1
fi
# remove unnecessary files
rm ${WORKING_DIR}/zeppelin/.gitignore
rm -rf ${WORKING_DIR}/zeppelin/.git
# create source package
cd ${WORKING_DIR}
cp -r zeppelin zeppelin-${RELEASE_NAME}
${TAR} cvzf zeppelin-${RELEASE_NAME}.tgz zeppelin-${RELEASE_NAME}
echo "Signing the source package"
cd ${WORKING_DIR}
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_NAME}.tgz.asc --detach-sig ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_NAME}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz.md5
${SHASUM} zeppelin-${RELEASE_NAME}.tgz > ${WORKING_DIR}/zeppelin-${RELEASE_NAME}.tgz.sha512
function make_binary_release() {
BIN_RELEASE_NAME="${1}"
BUILD_FLAGS="${2}"
cp -r ${WORKING_DIR}/zeppelin ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
cd ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
echo "mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}"
mvn clean package -Pbuild-distr -DskipTests ${BUILD_FLAGS}
if [[ $? -ne 0 ]]; then
echo "Build failed. ${BUILD_FLAGS}"
exit 1
fi
# re-create package with proper dir name
cd zeppelin-distribution/target
rm zeppelin-*.tar.gz
mv zeppelin-* zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
${TAR} cvzf zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
# sign bin package
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --armor --output zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.asc --detach-sig zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz
echo $GPG_PASSPHRASE | gpg --passphrase-fd 0 --print-md MD5 zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.md5
${SHASUM} zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz > zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.sha512
mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz ${WORKING_DIR}/
mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.asc ${WORKING_DIR}/
mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.md5 ${WORKING_DIR}/
mv zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}.tgz.sha512 ${WORKING_DIR}/
# clean up build dir
rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
}
make_binary_release spark-1.4.0_hadoop-2.3 -Pyarn -Pspark-1.4 -Dspark.version=1.4.0 -Phadoop-2.3
make_binary_release spark-1.3.1_hadoop-2.3 -Pyarn -Pspark-1.3 -Dspark.version=1.3.1 -Phadoop-2.3
# remove non release files and dirs
rm -rf ${WORKING_DIR}/zeppelin
rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_NAME}
echo "Release files are created under ${WORKING_DIR}"

53
dev/travis/save-logs.py Executable file
View file

@ -0,0 +1,53 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import subprocess
from datetime import datetime, timedelta
def main(file, cmd):
print cmd, "writing to", file
out = open(file, "w")
count = 0
process = subprocess.Popen(cmd,
stderr=subprocess.STDOUT,
stdout=subprocess.PIPE)
start = datetime.now()
nextPrint = datetime.now() + timedelta(seconds=1)
# wait for the process to terminate
pout = process.stdout
line = pout.readline()
while line:
count = count + 1
if datetime.now() > nextPrint:
diff = datetime.now() - start
sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count))
sys.stdout.flush()
nextPrint = datetime.now() + timedelta(seconds=10)
out.write(line)
line = pout.readline()
out.close()
errcode = process.wait()
diff = datetime.now() - start
sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count))
sys.stdout.write("\n" + str(cmd) + " done " + str(errcode) + "\n")
return errcode
if __name__ == "__main__":
if sys.argv < 1:
print "Usage: %s [file info]" % sys.argv[0]
sys.exit(1)
sys.exit(main(sys.argv[1], sys.argv[2:]))

31
dev/travis/travis-install.sh Executable file
View file

@ -0,0 +1,31 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
ZEPPELIN_SRC_ROOT_DIR=$1
shift
TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
cd ${ZEPPELIN_SRC_ROOT_DIR}
python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install.txt" "$@"
BUILD_RET_VAL=$?
if [[ "$BUILD_RET_VAL" != "0" ]];
then
cat "install.txt"
fi
exit ${BUILD_RET_VAL}

378
flink/pom.xml Normal file
View file

@ -0,0 +1,378 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-flink</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Flink</name>
<description>Zeppelin flink support</description>
<url>http://zeppelin.incubator.apache.org</url>
<properties>
<flink.version>0.9.0</flink.version>
<flink.akka.version>2.3.7</flink.akka.version>
<flink.scala.binary.version>2.10</flink.scala.binary.version>
<flink.scala.version>2.10.4</flink.scala.version>
<scala.macros.version>2.0.1</scala.macros.version>
</properties>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala-shell</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-actor_${flink.scala.binary.version}</artifactId>
<version>${flink.akka.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-remote_${flink.scala.binary.version}</artifactId>
<version>${flink.akka.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-slf4j_${flink.scala.binary.version}</artifactId>
<version>${flink.akka.version}</version>
</dependency>
<dependency>
<groupId>com.typesafe.akka</groupId>
<artifactId>akka-testkit_${flink.scala.binary.version}</artifactId>
<version>${flink.akka.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${flink.scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${flink.scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${flink.scala.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.gitignore</exclude>
<exclude>**/.settings/*</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
<exclude>**/target/**</exclude>
<exclude>**/README.md</exclude>
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>
</configuration>
</plugin>
<!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>3.1.4</version>
<executions>
<!-- Run scala compiler in the process-resources phase, so that dependencies on
scala classes can be resolved later in the (Java) compile phase -->
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
<!-- Run scala compiler in the process-test-resources phase, so that dependencies on
scala classes can be resolved later in the (Java) test-compile phase -->
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<jvmArgs>
<jvmArg>-Xms128m</jvmArg>
<jvmArg>-Xmx512m</jvmArg>
</jvmArgs>
<compilerPlugins combine.children="append">
<compilerPlugin>
<groupId>org.scalamacros</groupId>
<artifactId>paradise_${flink.scala.version}</artifactId>
<version>${scala.macros.version}</version>
</compilerPlugin>
</compilerPlugins>
</configuration>
</plugin>
<!-- Eclipse Integration -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-eclipse-plugin</artifactId>
<version>2.8</version>
<configuration>
<downloadSources>true</downloadSources>
<projectnatures>
<projectnature>org.scala-ide.sdt.core.scalanature</projectnature>
<projectnature>org.eclipse.jdt.core.javanature</projectnature>
</projectnatures>
<buildcommands>
<buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
</buildcommands>
<classpathContainers>
<classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
<classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
</classpathContainers>
<!-- excludes>
<exclude>org.scala-lang:scala-library</exclude>
<exclude>org.scala-lang:scala-compiler</exclude>
</excludes -->
<sourceIncludes>
<sourceInclude>**/*.scala</sourceInclude>
<sourceInclude>**/*.java</sourceInclude>
</sourceIncludes>
</configuration>
</plugin>
<!-- Adding scala source directories to build path -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.7</version>
<executions>
<!-- Add src/main/scala to eclipse build path -->
<execution>
<id>add-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>src/main/scala</source>
</sources>
</configuration>
</execution>
<!-- Add src/test/scala to eclipse build path -->
<execution>
<id>add-test-source</id>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>src/test/scala</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
<version>0.5.0</version>
<executions>
<execution>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
<configuration>
<verbose>false</verbose>
<failOnViolation>true</failOnViolation>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<failOnWarning>false</failOnWarning>
<sourceDirectory>${basedir}/src/main/scala</sourceDirectory>
<testSourceDirectory>${basedir}/src/test/scala</testSourceDirectory>
<configLocation>${project.basedir}/../_tools/scalastyle.xml</configLocation>
<outputFile>${project.basedir}/target/scalastyle-output.xml</outputFile>
<outputEncoding>UTF-8</outputEncoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/flink</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/flink</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

332
flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java Normal file
View file

@ -0,0 +1,332 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.flink;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.scala.FlinkILoop;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Console;
import scala.None;
import scala.Option;
import scala.Some;
import scala.tools.nsc.Settings;
import scala.tools.nsc.interpreter.IMain;
import scala.tools.nsc.settings.MutableSettings.BooleanSetting;
import scala.tools.nsc.settings.MutableSettings.PathSetting;
/**
* Interpreter for Apache Flink (http://flink.apache.org)
*/
public class FlinkInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(FlinkInterpreter.class);
private ByteArrayOutputStream out;
private Configuration flinkConf;
private LocalFlinkMiniCluster localFlinkCluster;
private FlinkILoop flinkIloop;
private Map<String, Object> binder;
private IMain imain;
public FlinkInterpreter(Properties property) {
super(property);
}
static {
Interpreter.register(
"flink",
"flink",
FlinkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("host", "local",
"host name of running JobManager. 'local' runs flink in local mode")
.add("port", "6123", "port of running JobManager")
.build()
);
}
@Override
public void open() {
out = new ByteArrayOutputStream();
flinkConf = new org.apache.flink.configuration.Configuration();
Properties intpProperty = getProperty();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));
flinkConf.setString(key, val);
}
if (localMode()) {
startFlinkMiniCluster();
}
flinkIloop = new FlinkILoop(getHost(), getPort(), (BufferedReader) null, new PrintWriter(out));
flinkIloop.settings_$eq(createSettings());
flinkIloop.createInterpreter();
imain = flinkIloop.intp();
// prepare bindings
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
binder = (Map<String, Object>) getValue("_binder");
// import libraries
imain.interpret("import scala.tools.nsc.io._");
imain.interpret("import Properties.userHome");
imain.interpret("import scala.compat.Platform.EOL");
imain.interpret("import org.apache.flink.api.scala._");
imain.interpret("import org.apache.flink.api.common.functions._");
imain.bindValue("env", flinkIloop.scalaEnv());
}
private boolean localMode() {
String host = getProperty("host");
return host == null || host.trim().length() == 0 || host.trim().equals("local");
}
private String getHost() {
if (localMode()) {
return "localhost";
} else {
return getProperty("host");
}
}
private int getPort() {
if (localMode()) {
return localFlinkCluster.getJobManagerRPCPort();
} else {
return Integer.parseInt(getProperty("port"));
}
}
private Settings createSettings() {
URL[] urls = getClassloaderUrls();
Settings settings = new Settings();
// set classpath
PathSetting pathSettings = settings.classpath();
String classpath = "";
List<File> paths = currentClassPath();
for (File f : paths) {
if (classpath.length() > 0) {
classpath += File.pathSeparator;
}
classpath += f.getAbsolutePath();
}
if (urls != null) {
for (URL u : urls) {
if (classpath.length() > 0) {
classpath += File.pathSeparator;
}
classpath += u.getFile();
}
}
pathSettings.v_$eq(classpath);
settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings);
settings.explicitParentLoader_$eq(new Some<ClassLoader>(Thread.currentThread()
.getContextClassLoader()));
BooleanSetting b = (BooleanSetting) settings.usejavacp();
b.v_$eq(true);
settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
return settings;
}
private List<File> currentClassPath() {
List<File> paths = classPath(Thread.currentThread().getContextClassLoader());
String[] cps = System.getProperty("java.class.path").split(File.pathSeparator);
if (cps != null) {
for (String cp : cps) {
paths.add(new File(cp));
}
}
return paths;
}
private List<File> classPath(ClassLoader cl) {
List<File> paths = new LinkedList<File>();
if (cl == null) {
return paths;
}
if (cl instanceof URLClassLoader) {
URLClassLoader ucl = (URLClassLoader) cl;
URL[] urls = ucl.getURLs();
if (urls != null) {
for (URL url : urls) {
paths.add(new File(url.getFile()));
}
}
}
return paths;
}
public Object getValue(String name) {
IMain imain = flinkIloop.intp();
Object ret = imain.valueOfTerm(name);
if (ret instanceof None) {
return null;
} else if (ret instanceof Some) {
return ((Some) ret).get();
} else {
return ret;
}
}
@Override
public void close() {
flinkIloop.closeInterpreter();
if (localMode()) {
stopFlinkMiniCluster();
}
}
@Override
public InterpreterResult interpret(String line, InterpreterContext context) {
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(Code.SUCCESS);
}
InterpreterResult result = interpret(line.split("\n"), context);
return result;
}
public InterpreterResult interpret(String[] lines, InterpreterContext context) {
IMain imain = flinkIloop.intp();
String[] linesToRun = new String[lines.length + 1];
for (int i = 0; i < lines.length; i++) {
linesToRun[i] = lines[i];
}
linesToRun[lines.length] = "print(\"\")";
Console.setOut(out);
out.reset();
Code r = null;
String incomplete = "";
for (int l = 0; l < linesToRun.length; l++) {
String s = linesToRun[l];
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
if (l + 1 < linesToRun.length) {
String nextLine = linesToRun[l + 1].trim();
if (nextLine.startsWith(".") && !nextLine.startsWith("..") && !nextLine.startsWith("./")) {
incomplete += s + "\n";
continue;
}
}
scala.tools.nsc.interpreter.Results.Result res = null;
try {
res = imain.interpret(incomplete + s);
} catch (Exception e) {
logger.info("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
r = getResultCode(res);
if (r == Code.ERROR) {
return new InterpreterResult(r, out.toString());
} else if (r == Code.INCOMPLETE) {
incomplete += s + "\n";
} else {
incomplete = "";
}
}
if (r == Code.INCOMPLETE) {
return new InterpreterResult(r, "Incomplete expression");
} else {
return new InterpreterResult(r, out.toString());
}
}
private Code getResultCode(scala.tools.nsc.interpreter.Results.Result r) {
if (r instanceof scala.tools.nsc.interpreter.Results.Success$) {
return Code.SUCCESS;
} else if (r instanceof scala.tools.nsc.interpreter.Results.Incomplete$) {
return Code.INCOMPLETE;
} else {
return Code.ERROR;
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public List<String> completion(String buf, int cursor) {
return new LinkedList<String>();
}
private void startFlinkMiniCluster() {
localFlinkCluster = new LocalFlinkMiniCluster(flinkConf, false);
localFlinkCluster.waitForTaskManagersToBeRegistered();
}
private void stopFlinkMiniCluster() {
if (localFlinkCluster != null) {
localFlinkCluster.stop();
localFlinkCluster = null;
}
}
static final String toString(Object o) {
return (o instanceof String) ? (String) o : "";
}
}

70
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java Normal file
View file

@ -0,0 +1,70 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.flink;
import static org.junit.Assert.assertEquals;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
public class FlinkInterpreterTest {
private static FlinkInterpreter flink;
private static InterpreterContext context;
@BeforeClass
public static void setUp() {
Properties p = new Properties();
flink = new FlinkInterpreter(p);
flink.open();
context = new InterpreterContext(null, null, null, null, null, null, null, null);
}
@AfterClass
public static void tearDown() {
flink.close();
flink.destroy();
}
@Test
public void testSimpleStatement() {
InterpreterResult result = flink.interpret("val a=1", context);
result = flink.interpret("print(a)", context);
assertEquals("1", result.message());
}
@Test
public void testNextlineInvoke() {
InterpreterResult result = flink.interpret("\"123\"\n .toInt", context);
assertEquals("res0: Int = 123\n", result.message());
}
@Test
public void testWordCount() {
flink.interpret("val text = env.fromElements(\"To be or not to be\")", context);
flink.interpret("val counts = text.flatMap { _.toLowerCase.split(\" \") }.map { (_, 1) }.groupBy(0).sum(1)", context);
InterpreterResult result = flink.interpret("counts.print()", context);
assertEquals(Code.SUCCESS, result.code());
}
}

151
geode/pom.xml Normal file
View file

@ -0,0 +1,151 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-geode</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Apache Geode interpreter</name>
<url>http://geode.incubator.apache.org/</url>
<properties>
<geode.version>1.0.0-incubating-SNAPSHOT</geode.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.geode</groupId>
<artifactId>gemfire-core</artifactId>
<version>${geode.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/geode</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/geode</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

319
geode/src/main/java/org/apache/zeppelin/geode/GeodeOqlInterpreter.java Normal file
View file

@ -0,0 +1,319 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.geode;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.gemstone.gemfire.cache.client.ClientCache;
import com.gemstone.gemfire.cache.client.ClientCacheFactory;
import com.gemstone.gemfire.cache.query.QueryService;
import com.gemstone.gemfire.cache.query.SelectResults;
import com.gemstone.gemfire.cache.query.Struct;
import com.gemstone.gemfire.pdx.PdxInstance;
/**
* Apache Geode OQL Interpreter (http://geode.incubator.apache.org)
*
* <ul>
* <li>{@code geode.locator.host} - The Geode Locator {@code <HOST>} to connect to.</li>
* <li>{@code geode.locator.port} - The Geode Locator {@code <PORT>} to connect to.</li>
* <li>{@code geode.max.result} - Max number of OQL result to display.</li>
* </ul>
* <p>
* Sample usages: <br/>
* {@code %geode.oql} <br/>
* {@code SELECT * FROM /regionEmployee e WHERE e.companyId > 95} <br/>
* {@code SELECT * FROM /regionEmployee ORDER BY employeeId} <br/>
* {@code
* SELECT * FROM /regionEmployee
* WHERE companyId IN SET(1, 3, 7) OR lastName IN SET('NameA', 'NameB')
* } <br/>
* {@code
* SELECT e.employeeId, c.id as companyId FROM /regionEmployee e, /regionCompany c
* WHERE e.companyId = c.id
* }
* </p>
* <p>
* OQL specification and sample queries:
* http://geode-docs.cfapps.io/docs/getting_started/querying_quick_reference.html
* </p>
* <p>
* When the Zeppelin server is collocated with Geode Shell (gfsh) one can use the %sh interpreter to
* run Geode shell commands: <br/>
* {@code
* %sh
* source /etc/geode/conf/geode-env.sh
* gfsh << EOF
* connect --locator=ambari.localdomain[10334]
* destroy region --name=/regionEmployee
* create region --name=regionEmployee --type=REPLICATE
* exit;
* EOF
*}
* </p>
* <p>
* Known issue:http://gemfire.docs.pivotal.io/bugnotes/KnownIssuesGemFire810.html #43673 Using query
* "select * from /exampleRegion.entrySet" fails in a client-server topology and/or in a
* PartitionedRegion.
* </p>
*/
public class GeodeOqlInterpreter extends Interpreter {
private Logger logger = LoggerFactory.getLogger(GeodeOqlInterpreter.class);
public static final String DEFAULT_PORT = "10334";
public static final String DEFAULT_HOST = "localhost";
public static final String DEFAULT_MAX_RESULT = "1000";
private static final char NEWLINE = '\n';
private static final char TAB = '\t';
private static final char WHITESPACE = ' ';
private static final String TABLE_MAGIC_TAG = "%table ";
public static final String LOCATOR_HOST = "geode.locator.host";
public static final String LOCATOR_PORT = "geode.locator.port";
public static final String MAX_RESULT = "geode.max.result";
static {
Interpreter.register(
"oql",
"geode",
GeodeOqlInterpreter.class.getName(),
new InterpreterPropertyBuilder().add(LOCATOR_HOST, DEFAULT_HOST, "The Geode Locator Host.")
.add(LOCATOR_PORT, DEFAULT_PORT, "The Geode Locator Port")
.add(MAX_RESULT, DEFAULT_MAX_RESULT, "Max number of OQL result to display.").build());
}
private ClientCache clientCache = null;
private QueryService queryService = null;
private Exception exceptionOnConnect;
private int maxResult;
public GeodeOqlInterpreter(Properties property) {
super(property);
}
protected ClientCache getClientCache() {
String locatorHost = getProperty(LOCATOR_HOST);
int locatorPort = Integer.valueOf(getProperty(LOCATOR_PORT));
ClientCache clientCache =
new ClientCacheFactory().addPoolLocator(locatorHost, locatorPort).create();
return clientCache;
}
@Override
public void open() {
logger.info("Geode open connection called!");
// Close the previous open connections.
close();
try {
maxResult = Integer.valueOf(getProperty(MAX_RESULT));
clientCache = getClientCache();
queryService = clientCache.getQueryService();
exceptionOnConnect = null;
logger.info("Successfully created Geode connection");
} catch (Exception e) {
logger.error("Cannot open connection", e);
exceptionOnConnect = e;
}
}
@Override
public void close() {
try {
if (clientCache != null) {
clientCache.close();
}
if (queryService != null) {
queryService.closeCqs();
}
} catch (Exception e) {
logger.error("Cannot close connection", e);
} finally {
clientCache = null;
queryService = null;
exceptionOnConnect = null;
}
}
private InterpreterResult executeOql(String oql) {
try {
if (getExceptionOnConnect() != null) {
return new InterpreterResult(Code.ERROR, getExceptionOnConnect().getMessage());
}
@SuppressWarnings("unchecked")
SelectResults<Object> results =
(SelectResults<Object>) getQueryService().newQuery(oql).execute();
StringBuilder msg = new StringBuilder(TABLE_MAGIC_TAG);
boolean isTableHeaderSet = false;
Iterator<Object> iterator = results.iterator();
int rowDisplayCount = 0;
while (iterator.hasNext() && (rowDisplayCount < getMaxResult())) {
Object entry = iterator.next();
rowDisplayCount++;
if (entry instanceof Number) {
handleNumberEntry(isTableHeaderSet, entry, msg);
} else if (entry instanceof Struct) {
handleStructEntry(isTableHeaderSet, entry, msg);
} else if (entry instanceof PdxInstance) {
handlePdxInstanceEntry(isTableHeaderSet, entry, msg);
} else {
handleUnsupportedTypeEntry(isTableHeaderSet, entry, msg);
}
isTableHeaderSet = true;
msg.append(NEWLINE);
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
} catch (Exception ex) {
logger.error("Cannot run " + oql, ex);
return new InterpreterResult(Code.ERROR, ex.getMessage());
}
}
/**
* Zeppelin's %TABLE convention uses tab (\t) to delimit fields and new-line (\n) to delimit rows
* To complain with this convention we need to replace any occurrences of tab and/or newline
* characters in the content.
*/
private String replaceReservedChars(String str) {
if (StringUtils.isBlank(str)) {
return str;
}
return str.replace(TAB, WHITESPACE).replace(NEWLINE, WHITESPACE);
}
private void handleStructEntry(boolean isHeaderSet, Object entry, StringBuilder msg) {
Struct struct = (Struct) entry;
if (!isHeaderSet) {
for (String titleName : struct.getStructType().getFieldNames()) {
msg.append(replaceReservedChars(titleName)).append(TAB);
}
msg.append(NEWLINE);
}
for (String titleName : struct.getStructType().getFieldNames()) {
msg.append(replaceReservedChars("" + struct.get(titleName))).append(TAB);
}
}
private void handlePdxInstanceEntry(boolean isHeaderSet, Object entry, StringBuilder msg) {
PdxInstance pdxEntry = (PdxInstance) entry;
if (!isHeaderSet) {
for (String titleName : pdxEntry.getFieldNames()) {
msg.append(replaceReservedChars(titleName)).append(TAB);
}
msg.append(NEWLINE);
}
for (String titleName : pdxEntry.getFieldNames()) {
msg.append(replaceReservedChars("" + pdxEntry.getField(titleName))).append(TAB);
}
}
private void handleNumberEntry(boolean isHeaderSet, Object entry, StringBuilder msg) {
if (!isHeaderSet) {
msg.append("Result").append(NEWLINE);
}
msg.append((Number) entry);
}
private void handleUnsupportedTypeEntry(boolean isHeaderSet, Object entry, StringBuilder msg) {
if (!isHeaderSet) {
msg.append("Unsuppoted Type").append(NEWLINE);
}
msg.append("" + entry);
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.info("Run OQL command '{}'", cmd);
return executeOql(cmd);
}
@Override
public void cancel(InterpreterContext context) {
// Do nothing
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
GeodeOqlInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
public int getMaxResult() {
return maxResult;
}
// Test only
QueryService getQueryService() {
return this.queryService;
}
Exception getExceptionOnConnect() {
return this.exceptionOnConnect;
}
}

181
geode/src/test/java/org/apache/zeppelin/geode/GeodeOqlInterpreterTest.java Normal file
View file

@ -0,0 +1,181 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.geode;
import static org.apache.zeppelin.geode.GeodeOqlInterpreter.*;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter.FormType;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.Test;
import com.gemstone.gemfire.cache.query.QueryService;
import com.gemstone.gemfire.cache.query.SelectResults;
import com.gemstone.gemfire.cache.query.Struct;
import com.gemstone.gemfire.cache.query.internal.StructImpl;
import com.gemstone.gemfire.cache.query.internal.types.StructTypeImpl;
import com.gemstone.gemfire.pdx.PdxInstance;
import com.gemstone.gemfire.pdx.internal.PdxInstanceImpl;
import com.gemstone.gemfire.pdx.internal.PdxType;
public class GeodeOqlInterpreterTest {
private static final String OQL_QUERY = "select * from /region";
private static Iterator<Object> asIterator(Object... items) {
return new ArrayList<Object>(Arrays.asList(items)).iterator();
}
@Test
public void testOpenCommandIndempotency() {
Properties properties = new Properties();
properties.put(LOCATOR_HOST, DEFAULT_HOST);
properties.put(LOCATOR_PORT, DEFAULT_PORT);
properties.put(MAX_RESULT, DEFAULT_MAX_RESULT);
GeodeOqlInterpreter spyGeodeOqlInterpreter = spy(new GeodeOqlInterpreter(properties));
// Ensure that an attempt to open new connection will clean any remaining connections
spyGeodeOqlInterpreter.open();
spyGeodeOqlInterpreter.open();
spyGeodeOqlInterpreter.open();
verify(spyGeodeOqlInterpreter, times(3)).open();
verify(spyGeodeOqlInterpreter, times(3)).close();
}
@Test
public void oqlNumberResponse() throws Exception {
testOql(asIterator(66, 67), "Result\n66\n67\n", 10);
testOql(asIterator(66, 67), "Result\n66\n", 1);
}
@Test
public void oqlStructResponse() throws Exception {
String[] fields = new String[] {"field1", "field2"};
Struct s1 = new StructImpl(new StructTypeImpl(fields), new String[] {"val11", "val12"});
Struct s2 = new StructImpl(new StructTypeImpl(fields), new String[] {"val21", "val22"});
testOql(asIterator(s1, s2), "field1\tfield2\t\nval11\tval12\t\nval21\tval22\t\n", 10);
testOql(asIterator(s1, s2), "field1\tfield2\t\nval11\tval12\t\n", 1);
}
@Test
public void oqlStructResponseWithReservedCharacters() throws Exception {
String[] fields = new String[] {"fi\teld1", "f\nield2"};
Struct s1 = new StructImpl(new StructTypeImpl(fields), new String[] {"v\nal\t1", "val2"});
testOql(asIterator(s1), "fi eld1\tf ield2\t\nv al 1\tval2\t\n", 10);
}
@Test
public void oqlPdxInstanceResponse() throws Exception {
ByteArrayInputStream bais = new ByteArrayInputStream("koza\tboza\n".getBytes());
PdxInstance pdx1 = new PdxInstanceImpl(new PdxType(), new DataInputStream(bais), 4);
PdxInstance pdx2 = new PdxInstanceImpl(new PdxType(), new DataInputStream(bais), 4);
testOql(asIterator(pdx1, pdx2), "\n\n\n", 10);
testOql(asIterator(pdx1, pdx2), "\n\n", 1);
}
private static class DummyUnspportedType {
@Override
public String toString() {
return "Unsupported Indeed";
}
}
@Test
public void oqlUnsupportedTypeResponse() throws Exception {
DummyUnspportedType unspported1 = new DummyUnspportedType();
DummyUnspportedType unspported2 = new DummyUnspportedType();
testOql(asIterator(unspported1, unspported2), "Unsuppoted Type\n" + unspported1.toString()
+ "\n" + unspported1.toString() + "\n", 10);
}
private void testOql(Iterator<Object> queryResponseIterator, String expectedOutput, int maxResult)
throws Exception {
GeodeOqlInterpreter spyGeodeOqlInterpreter = spy(new GeodeOqlInterpreter(new Properties()));
QueryService mockQueryService = mock(QueryService.class, RETURNS_DEEP_STUBS);
when(spyGeodeOqlInterpreter.getQueryService()).thenReturn(mockQueryService);
when(spyGeodeOqlInterpreter.getMaxResult()).thenReturn(maxResult);
@SuppressWarnings("unchecked")
SelectResults<Object> mockResults = mock(SelectResults.class);
when(mockQueryService.newQuery(eq(OQL_QUERY)).execute()).thenReturn(mockResults);
when(mockResults.iterator()).thenReturn(queryResponseIterator);
InterpreterResult interpreterResult = spyGeodeOqlInterpreter.interpret(OQL_QUERY, null);
assertEquals(Code.SUCCESS, interpreterResult.code());
assertEquals(expectedOutput, interpreterResult.message());
}
@Test
public void oqlWithQueryException() throws Exception {
GeodeOqlInterpreter spyGeodeOqlInterpreter = spy(new GeodeOqlInterpreter(new Properties()));
when(spyGeodeOqlInterpreter.getExceptionOnConnect()).thenReturn(
new RuntimeException("Test Exception On Connect"));
InterpreterResult interpreterResult = spyGeodeOqlInterpreter.interpret(OQL_QUERY, null);
assertEquals(Code.ERROR, interpreterResult.code());
assertEquals("Test Exception On Connect", interpreterResult.message());
}
@Test
public void oqlWithExceptionOnConnect() throws Exception {
GeodeOqlInterpreter spyGeodeOqlInterpreter = spy(new GeodeOqlInterpreter(new Properties()));
when(spyGeodeOqlInterpreter.getQueryService()).thenThrow(
new RuntimeException("Expected Test Exception!"));
InterpreterResult interpreterResult = spyGeodeOqlInterpreter.interpret(OQL_QUERY, null);
assertEquals(Code.ERROR, interpreterResult.code());
assertEquals("Expected Test Exception!", interpreterResult.message());
}
@Test
public void testFormType() {
assertEquals(FormType.SIMPLE, new GeodeOqlInterpreter(new Properties()).getFormType());
}
}

2
hbase/pom.xml
View file

@ -36,6 +36,8 @@
<properties>
<hbase.hbase.version>1.0.0</hbase.hbase.version>
<hbase.hadoop.version>2.6.0</hbase.hadoop.version>
<jruby.version>1.6.8</jruby.version>
<protobuf.version>2.4.1</protobuf.version>
</properties>
<dependencies>

5
hive/pom.xml
View file

@ -22,13 +22,14 @@
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.5.0-incubating-SNAPSHOT</version>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-hive</artifactId>
<packaging>jar</packaging>
<version>0.5.0-incubating-SNAPSHOT</version>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Hive interpreter</name>
<url>http://www.apache.org</url>

19
hive/src/main/java/org/apache/zeppelin/hive/HiveInterpreter.java
View file

@ -17,18 +17,25 @@
*/
package org.apache.zeppelin.hive;
import java.sql.*;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.*;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Hive interpreter for Zeppelin.
@ -43,7 +50,7 @@ public class HiveInterpreter extends Interpreter {
static {
Interpreter.register(
"hive",
"hql",
"hive",
HiveInterpreter.class.getName(),
new InterpreterPropertyBuilder()

175
ignite/pom.xml Normal file
View file

@ -0,0 +1,175 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<artifactId>zeppelin-ignite</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Apache Ignite interpreter</name>
<url>http://zeppelin.incubator.apache.org</url>
<properties>
<ignite.version>1.1.0-incubating</ignite.version>
<ignite.scala.binary.version>2.10</ignite.scala.binary.version>
<ignite.scala.version>2.10.4</ignite.scala.version>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-core</artifactId>
<version>${ignite.version}</version>
</dependency>
<dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-spring</artifactId>
<version>${ignite.version}</version>
</dependency>
<dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-indexing</artifactId>
<version>${ignite.version}</version>
</dependency>
<dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-scalar</artifactId>
<version>${ignite.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${ignite.scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${ignite.scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${ignite.scala.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/ignite</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/ignite</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

353
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java Normal file
View file

@ -0,0 +1,353 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.ignite;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.LinkedList;
import java.util.Map;
import java.util.Properties;
import scala.Console;
import scala.None;
import scala.Some;
import scala.tools.nsc.Settings;
import scala.tools.nsc.interpreter.IMain;
import scala.tools.nsc.interpreter.Results.Result;
import scala.tools.nsc.settings.MutableSettings.BooleanSetting;
import scala.tools.nsc.settings.MutableSettings.PathSetting;
/**
* Apache Ignite interpreter (http://ignite.incubator.apache.org/).
*
* Use the following properties for interpreter configuration:
*
* <ul>
* <li>{@code ignite.addresses} - coma separated list of hosts in form {@code <host>:<port>}
* or {@code <host>:<port_1>..<port_n>} </li>
* <li>{@code ignite.clientMode} - indicates that Ignite interpreter
* should start node in client mode ({@code true} or {@code false}).</li>
* <li>{@code ignite.peerClassLoadingEnabled} - enables/disables peer class loading
* ({@code true} or {@code false}).</li>
* <li>{@code ignite.config.url} - URL for Ignite configuration. If this URL specified then
* all aforementioned properties will not be taken in account.</li>
* </ul>
*/
public class IgniteInterpreter extends Interpreter {
static final String IGNITE_ADDRESSES = "ignite.addresses";
static final String IGNITE_CLIENT_MODE = "ignite.clientMode";
static final String IGNITE_PEER_CLASS_LOADING_ENABLED = "ignite.peerClassLoadingEnabled";
static final String IGNITE_CFG_URL = "ignite.config.url";
static {
Interpreter.register(
"ignite",
"ignite",
IgniteInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(IGNITE_ADDRESSES, "127.0.0.1:47500..47509",
"Coma separated list of addresses "
+ "(e.g. 127.0.0.1:47500 or 127.0.0.1:47500..47509)")
.add(IGNITE_CLIENT_MODE, "true", "Client mode. true or false")
.add(IGNITE_CFG_URL, "", "Configuration URL. Overrides all other settings.")
.add(IGNITE_PEER_CLASS_LOADING_ENABLED, "true",
"Peer class loading enabled. true or false")
.build());
}
private Logger logger = LoggerFactory.getLogger(IgniteInterpreter.class);
private Ignite ignite;
private ByteArrayOutputStream out;
private IMain imain;
private Throwable initEx;
public IgniteInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
Settings settings = new Settings();
URL[] urls = getClassloaderUrls();
// set classpath
PathSetting pathSettings = settings.classpath();
StringBuilder sb = new StringBuilder();
for (File f : currentClassPath()) {
if (sb.length() > 0) {
sb.append(File.pathSeparator);
}
sb.append(f.getAbsolutePath());
}
if (urls != null) {
for (URL u : urls) {
if (sb.length() > 0) {
sb.append(File.pathSeparator);
}
sb.append(u.getFile());
}
}
pathSettings.v_$eq(sb.toString());
settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings);
settings.explicitParentLoader_$eq(new Some<>(Thread.currentThread().getContextClassLoader()));
BooleanSetting b = (BooleanSetting) settings.usejavacp();
b.v_$eq(true);
settings.scala$tools$nsc$settings$StandardScalaSettings$_setter_$usejavacp_$eq(b);
out = new ByteArrayOutputStream();
imain = new IMain(settings, new PrintWriter(out));
initIgnite();
}
private List<File> currentClassPath() {
List<File> paths = classPath(Thread.currentThread().getContextClassLoader());
String[] cps = System.getProperty("java.class.path").split(File.pathSeparator);
for (String cp : cps) {
paths.add(new File(cp));
}
return paths;
}
private List<File> classPath(ClassLoader cl) {
List<File> paths = new LinkedList<>();
if (cl == null) {
return paths;
}
if (cl instanceof URLClassLoader) {
URLClassLoader ucl = (URLClassLoader) cl;
URL[] urls = ucl.getURLs();
if (urls != null) {
for (URL url : urls) {
paths.add(new File(url.getFile()));
}
}
}
return paths;
}
public Object getValue(String name) {
Object val = imain.valueOfTerm(name);
if (val instanceof None) {
return null;
} else if (val instanceof Some) {
return ((Some) val).get();
} else {
return val;
}
}
private Ignite getIgnite() {
if (ignite == null) {
try {
String cfgUrl = getProperty(IGNITE_CFG_URL);
if (cfgUrl != null && !cfgUrl.isEmpty()) {
ignite = Ignition.start(new URL(cfgUrl));
} else {
IgniteConfiguration conf = new IgniteConfiguration();
conf.setClientMode(Boolean.parseBoolean(getProperty(IGNITE_CLIENT_MODE)));
TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
ipFinder.setAddresses(getAddresses());
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
discoSpi.setIpFinder(ipFinder);
conf.setDiscoverySpi(discoSpi);
conf.setPeerClassLoadingEnabled(
Boolean.parseBoolean(getProperty(IGNITE_PEER_CLASS_LOADING_ENABLED)));
ignite = Ignition.start(conf);
}
initEx = null;
} catch (Exception e) {
initEx = e;
}
}
return ignite;
}
private void initIgnite() {
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder = (Map<String, Object>) getValue("_binder");
if (getIgnite() != null) {
binder.put("ignite", ignite);
imain.interpret("@transient val ignite = "
+ "_binder.get(\"ignite\")"
+ ".asInstanceOf[org.apache.ignite.Ignite]");
}
}
@Override
public void close() {
initEx = null;
if (ignite != null) {
ignite.close();
ignite = null;
}
if (imain != null) {
imain.close();
imain = null;
}
}
private List<String> getAddresses() {
String prop = getProperty(IGNITE_ADDRESSES);
if (prop == null || prop.isEmpty()) {
return Collections.emptyList();
}
String[] tokens = prop.split(",");
List<String> addresses = new ArrayList<>(tokens.length);
Collections.addAll(addresses, tokens);
return addresses;
}
@Override
public InterpreterResult interpret(String line, InterpreterContext context) {
if (initEx != null) {
return IgniteInterpreterUtils.buildErrorResult(initEx);
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(Code.SUCCESS);
}
return interpret(line.split("\n"));
}
@Override
public void cancel(InterpreterContext context) {
}
private InterpreterResult interpret(String[] lines) {
String[] linesToRun = new String[lines.length + 1];
System.arraycopy(lines, 0, linesToRun, 0, lines.length);
linesToRun[lines.length] = "print(\"\")";
Console.setOut(out);
out.reset();
Code code = null;
String incomplete = "";
for (int l = 0; l < linesToRun.length; l++) {
String s = linesToRun[l];
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
if (l + 1 < linesToRun.length) {
String nextLine = linesToRun[l + 1].trim();
if (nextLine.startsWith(".") && !nextLine.startsWith("..") && !nextLine.startsWith("./")) {
incomplete += s + "\n";
continue;
}
}
try {
code = getResultCode(imain.interpret(incomplete + s));
} catch (Exception e) {
logger.info("Interpreter exception", e);
return new InterpreterResult(Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
if (code == Code.ERROR) {
return new InterpreterResult(code, out.toString());
} else if (code == Code.INCOMPLETE) {
incomplete += s + '\n';
} else {
incomplete = "";
}
}
if (code == Code.INCOMPLETE) {
return new InterpreterResult(code, "Incomplete expression");
} else {
return new InterpreterResult(code, out.toString());
}
}
private Code getResultCode(Result res) {
if (res instanceof scala.tools.nsc.interpreter.Results.Success$) {
return Code.SUCCESS;
} else if (res instanceof scala.tools.nsc.interpreter.Results.Incomplete$) {
return Code.INCOMPLETE;
} else {
return Code.ERROR;
}
}
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public List<String> completion(String buf, int cursor) {
return new LinkedList<>();
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
IgniteInterpreter.class.getName() + this.hashCode());
}
}

44
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreterUtils.java Normal file
View file

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.ignite;
import org.apache.zeppelin.interpreter.InterpreterResult;
/**
* Apache Ignite interpreter utils.
*/
public class IgniteInterpreterUtils {
/**
* Builds error result from given exception.
* @param e Exception.
* @return result.
*/
public static InterpreterResult buildErrorResult(Throwable e) {
StringBuilder sb = new StringBuilder(e.getMessage());
while ((e = e.getCause()) != null) {
String errMsg = e.getMessage();
if (errMsg != null) {
sb.append('\n').append(errMsg);
}
}
return new InterpreterResult(InterpreterResult.Code.ERROR, sb.toString());
}
}

197
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java Normal file
View file

@ -0,0 +1,197 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.ignite;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
/**
* Apache Ignite SQL interpreter (http://ignite.incubator.apache.org/).
*
* Use {@code ignite.jdbc.url} property to set up JDBC connection URL.
* URL has the following pattern:
* {@code jdbc:ignite://<hostname>:<port>/<cache_name>}
*
* <ul>
* <li>Hostname is required.</li>
* <li>If port is not defined, 11211 is used (default for Ignite client).</li>
* <li>Leave cache_name empty if you are connecting to a default cache.
* Note that the cache name is case sensitive.</li>
* </ul>
*/
public class IgniteSqlInterpreter extends Interpreter {
private static final String IGNITE_JDBC_DRIVER_NAME = "org.apache.ignite.IgniteJdbcDriver";
static final String IGNITE_JDBC_URL = "ignite.jdbc.url";
static {
Interpreter.register(
"ignitesql",
"ignite",
IgniteSqlInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(IGNITE_JDBC_URL, "jdbc:ignite://localhost:11211/", "Ignite JDBC connection URL.")
.build());
}
private Logger logger = LoggerFactory.getLogger(IgniteSqlInterpreter.class);
private Connection conn;
private Throwable connEx;
private Statement curStmt;
public IgniteSqlInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
try {
Class.forName(IGNITE_JDBC_DRIVER_NAME);
} catch (ClassNotFoundException e) {
logger.error("Can't open connection", e);
connEx = e;
return;
}
try {
logger.info("connect to " + getProperty(IGNITE_JDBC_URL));
conn = DriverManager.getConnection(getProperty(IGNITE_JDBC_URL));
connEx = null;
logger.info("Successfully created JDBC connection");
} catch (SQLException e) {
logger.error("Can't open connection: ", e);
connEx = e;
}
}
@Override
public void close() {
try {
if (conn != null) {
conn.close();
}
} catch (SQLException e) {
throw new InterpreterException(e);
} finally {
conn = null;
connEx = null;
}
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
if (connEx != null) {
return new InterpreterResult(Code.ERROR, connEx.getMessage());
}
StringBuilder msg = new StringBuilder("%table ");
try (Statement stmt = conn.createStatement()) {
curStmt = stmt;
try (ResultSet res = stmt.executeQuery(st)) {
ResultSetMetaData md = res.getMetaData();
for (int i = 1; i <= md.getColumnCount(); i++) {
if (i > 1) {
msg.append('\t');
}
msg.append(md.getColumnName(i));
}
msg.append('\n');
while (res.next()) {
for (int i = 1; i <= md.getColumnCount(); i++) {
msg.append(res.getString(i));
if (i != md.getColumnCount()) {
msg.append('\t');
}
}
msg.append('\n');
}
}
} catch (Exception e) {
return IgniteInterpreterUtils.buildErrorResult(e);
} finally {
curStmt = null;
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
}
@Override
public void cancel(InterpreterContext context) {
if (curStmt != null) {
try {
curStmt.cancel();
} catch (SQLException e) {
// No-op.
} finally {
curStmt = null;
}
}
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
IgniteSqlInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return new LinkedList<>();
}
}

99
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java Normal file
View file

@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.ignite;
import java.util.Collections;
import java.util.Properties;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Tests for Apache Ignite interpreter ({@link IgniteInterpreter}).
*/
public class IgniteInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null);
private IgniteInterpreter intp;
private Ignite ignite;
@Before
public void setUp() {
TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
ipFinder.setAddresses(Collections.singletonList(HOST));
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
discoSpi.setIpFinder(ipFinder);
IgniteConfiguration cfg = new IgniteConfiguration();
cfg.setDiscoverySpi(discoSpi);
cfg.setGridName("test");
ignite = Ignition.start(cfg);
Properties props = new Properties();
props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL, "jdbc:intp://localhost:11211/person");
props.setProperty(IgniteInterpreter.IGNITE_CLIENT_MODE, "false");
props.setProperty(IgniteInterpreter.IGNITE_PEER_CLASS_LOADING_ENABLED, "false");
intp = new IgniteInterpreter(props);
intp.open();
}
@After
public void tearDown() {
ignite.close();
intp.close();
}
@Test
public void testInterpret() {
String sizeVal = "size";
InterpreterResult result = intp.interpret("import org.apache.ignite.IgniteCache\n" +
"val " + sizeVal + " = ignite.cluster().nodes().size()", INTP_CONTEXT);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(result.message().contains(sizeVal + ": Int = " + ignite.cluster().nodes().size()));
result = intp.interpret("\"123\"\n .toInt", INTP_CONTEXT);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
}
@Test
public void testInterpretInvalidInput() {
InterpreterResult result = intp.interpret("invalid input", INTP_CONTEXT);
assertEquals(InterpreterResult.Code.ERROR, result.code());
}
}

105
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java Normal file
View file

@ -0,0 +1,105 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.ignite;
import java.util.Collections;
import java.util.Properties;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
/**
* Tests for Apache Ignite SQL interpreter ({@link IgniteSqlInterpreter}).
*/
public class IgniteSqlInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null);
private Ignite ignite;
private IgniteSqlInterpreter intp;
@Before
public void setUp() {
TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
ipFinder.setAddresses(Collections.singletonList(HOST));
TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
discoSpi.setIpFinder(ipFinder);
IgniteConfiguration cfg = new IgniteConfiguration();
cfg.setDiscoverySpi(discoSpi);
cfg.setGridName("test");
ignite = Ignition.start(cfg);
Properties props = new Properties();
props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL, "jdbc:ignite://localhost:11211/person");
props.setProperty(IgniteInterpreter.IGNITE_CLIENT_MODE, "false");
intp = new IgniteSqlInterpreter(props);
CacheConfiguration<Integer, Person> cacheConf = new CacheConfiguration<>();
cacheConf.setIndexedTypes(Integer.class, Person.class);
cacheConf.setName("person");
IgniteCache<Integer, Person> cache = ignite.createCache(cacheConf);
cache.put(1, new Person("sun", 100));
cache.put(2, new Person("moon", 50));
assertEquals("moon", cache.get(2).getName());
intp.open();
}
@After
public void tearDown() {
intp.close();
ignite.close();
}
@Test
public void testSql() {
InterpreterResult result = intp.interpret("select name, age from person where age > 10", INTP_CONTEXT);
assertEquals(Code.SUCCESS, result.code());
assertEquals(Type.TABLE, result.type());
assertEquals("NAME\tAGE\nsun\t100\nmoon\t50\n", result.message());
}
@Test
public void testInvalidSql() throws Exception {
InterpreterResult result = intp.interpret("select * hrom person", INTP_CONTEXT);
assertEquals(Code.ERROR, result.code());
}
}

50
ignite/src/test/java/org/apache/zeppelin/ignite/Person.java Normal file
View file

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.ignite;
import java.io.Serializable;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
public class Person implements Serializable {
@QuerySqlField
private String name;
@QuerySqlField
private int age;
public Person(String name, int age) {
this.name = name;
this.age = age;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public int getAge() {
return age;
}
public void setAge(int age) {
this.age = age;
}
}

130
kylin/pom.xml Executable file
View file

@ -0,0 +1,130 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-kylin</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Kylin interpreter</name>
<url>http://zeppelin.incubator.apache.org</url>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.5</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/kylin</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/kylin</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

201
kylin/src/main/java/org/apache/zeppelin/kylin/KylinInterpreter.java Executable file
View file

@ -0,0 +1,201 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.kylin;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Kylin interpreter for Zeppelin. (http://kylin.io)
*/
public class KylinInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(KylinInterpreter.class);
static final String KYLIN_QUERY_API_URL = "kylin.api.url";
static final String KYLIN_USERNAME = "kylin.api.user";
static final String KYLIN_PASSWORD = "kylin.api.password";
static final String KYLIN_QUERY_PROJECT = "kylin.query.project";
static final String KYLIN_QUERY_OFFSET = "kylin.query.offset";
static final String KYLIN_QUERY_LIMIT = "kylin.query.limit";
static final String KYLIN_QUERY_ACCEPT_PARTIAL = "kylin.query.ispartial";
static final Pattern KYLIN_TABLE_FORMAT_REGEX_LABEL = Pattern.compile("\"label\":\"(.*?)\"");
static final Pattern KYLIN_TABLE_FORMAT_REGEX = Pattern.compile("\"results\":\\[\\[\"(.*?)\"]]");
static {
Interpreter.register(
"kylin",
"kylin",
KylinInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(KYLIN_USERNAME, "ADMIN", "username for kylin user")
.add(KYLIN_PASSWORD, "KYLIN", "password for kylin user")
.add(KYLIN_QUERY_API_URL, "http://<host>:<port>/kylin/api/query", "Kylin API.")
.add(KYLIN_QUERY_PROJECT, "default", "kylin project name")
.add(KYLIN_QUERY_OFFSET, "0", "kylin query offset")
.add(KYLIN_QUERY_LIMIT, "5000", "kylin query limit")
.add(KYLIN_QUERY_ACCEPT_PARTIAL, "true", "The kylin query partial flag").build());
}
public KylinInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
try {
return executeQuery(st);
} catch (IOException e) {
logger.error("failed to query data in kylin ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
public HttpResponse prepareRequest(String sql) throws IOException {
String KYLIN_PROJECT = getProperty(KYLIN_QUERY_PROJECT);
logger.info("project:" + KYLIN_PROJECT);
logger.info("sql:" + sql);
logger.info("acceptPartial:" + getProperty(KYLIN_QUERY_ACCEPT_PARTIAL));
logger.info("limit:" + getProperty(KYLIN_QUERY_LIMIT));
logger.info("offset:" + getProperty(KYLIN_QUERY_OFFSET));
byte[] encodeBytes = Base64.encodeBase64(new String(getProperty(KYLIN_USERNAME)
+ ":" + getProperty(KYLIN_PASSWORD)).getBytes("UTF-8"));
String postContent = new String("{\"project\":" + "\"" + KYLIN_PROJECT + "\""
+ "," + "\"sql\":" + "\"" + sql + "\""
+ "," + "\"acceptPartial\":" + "\"" + getProperty(KYLIN_QUERY_ACCEPT_PARTIAL) + "\""
+ "," + "\"offset\":" + "\"" + getProperty(KYLIN_QUERY_OFFSET) + "\""
+ "," + "\"limit\":" + "\"" + getProperty(KYLIN_QUERY_LIMIT) + "\"" + "}");
logger.info("post:" + postContent);
postContent = postContent.replaceAll("[\u0000-\u001f]", " ");
StringEntity entity = new StringEntity(postContent, "UTF-8");
entity.setContentType("application/json; charset=UTF-8");
logger.info("post url:" + getProperty(KYLIN_QUERY_API_URL));
HttpPost postRequest = new HttpPost(getProperty(KYLIN_QUERY_API_URL));
postRequest.setEntity(entity);
postRequest.addHeader("Authorization", "Basic " + new String(encodeBytes));
postRequest.addHeader("Accept-Encoding", "UTF-8");
HttpClient httpClient = HttpClientBuilder.create().build();
return httpClient.execute(postRequest);
}
private InterpreterResult executeQuery(String sql) throws IOException {
HttpResponse response = prepareRequest(sql);
if (response.getStatusLine().getStatusCode() != 200) {
logger.error("failed to execute query: " + response.getEntity().getContent().toString());
return new InterpreterResult(InterpreterResult.Code.ERROR,
"Failed : HTTP error code " + response.getStatusLine().getStatusCode());
}
BufferedReader br = new BufferedReader(
new InputStreamReader((response.getEntity().getContent())));
StringBuilder sb = new StringBuilder();
String output;
logger.info("Output from Server .... \n");
while ((output = br.readLine()) != null) {
logger.info(output);
sb.append(output).append('\n');
}
InterpreterResult rett = new InterpreterResult(InterpreterResult.Code.SUCCESS,
formatResult(sb.toString()));
return rett;
}
private String formatResult(String msg) {
StringBuilder res = new StringBuilder("%table ");
Matcher ml = KYLIN_TABLE_FORMAT_REGEX_LABEL.matcher(msg);
while (!ml.hitEnd() && ml.find()) {
res.append(ml.group(1) + " \t");
}
res.append(" \n");
Matcher mr = KYLIN_TABLE_FORMAT_REGEX.matcher(msg);
String table = null;
while (!mr.hitEnd() && mr.find()) {
table = mr.group(1);
}
String[] row = table.split("\"],\\[\"");
for (int i = 0; i < row.length; i++) {
String[] col = row[i].split("\",\"");
for (int j = 0; j < col.length; j++) {
res.append(col[j] + " \t");
}
res.append(" \n");
}
return res.toString();
}
}

201
kylin/src/test/java/KylinInterpreterTest.java Executable file
View file

@ -0,0 +1,201 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.http.*;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.AbstractHttpMessage;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.kylin.KylinInterpreter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Locale;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
public class KylinInterpreterTest {
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@Test
public void test(){
KylinInterpreter t = new MockKylinInterpreter(new Properties());
InterpreterResult result = t.interpret(
"select a.date,sum(b.measure) as measure from kylin_fact_table a " +
"inner join kylin_lookup_table b on a.date=b.date group by a.date", null);
assertEquals(InterpreterResult.Type.TABLE,result.type());
}
}
class MockKylinInterpreter extends KylinInterpreter {
public MockKylinInterpreter(Properties property) {
super(property);
}
@Override
public HttpResponse prepareRequest(String sql) throws IOException {
MockHttpClient client = new MockHttpClient();
return client.execute(new HttpPost());
}
}
class MockHttpClient{
public MockHttpResponse execute(HttpPost post){
return new MockHttpResponse();
}
}
class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
@Override
public StatusLine getStatusLine() {
return new MockStatusLine();
}
@Override
public void setStatusLine(StatusLine statusLine) {
}
@Override
public void setStatusLine(ProtocolVersion protocolVersion, int i) {
}
@Override
public void setStatusLine(ProtocolVersion protocolVersion, int i, String s) {
}
@Override
public void setStatusCode(int i) throws IllegalStateException {
}
@Override
public void setReasonPhrase(String s) throws IllegalStateException {
}
@Override
public HttpEntity getEntity() {
return new MockEntity();
}
@Override
public void setEntity(HttpEntity httpEntity) {
}
@Override
public Locale getLocale() {
return null;
}
@Override
public void setLocale(Locale locale) {
}
@Override
public ProtocolVersion getProtocolVersion() {
return null;
}
}
class MockStatusLine implements StatusLine{
@Override
public ProtocolVersion getProtocolVersion() {
return null;
}
@Override
public int getStatusCode() {
return 200;
}
@Override
public String getReasonPhrase() {
return null;
}
}
class MockEntity implements HttpEntity{
@Override
public boolean isRepeatable() {
return false;
}
@Override
public boolean isChunked() {
return false;
}
@Override
public long getContentLength() {
return 0;
}
@Override
public Header getContentType() {
return null;
}
@Override
public Header getContentEncoding() {
return null;
}
@Override
public InputStream getContent() throws IOException, IllegalStateException {
return new ByteArrayInputStream(("{\"columnMetas\":" +
"[{\"label\":\"PART_DT\"},{\"label\":\"measure\"}]," +
"\"results\":[[\"2012-01-03\",\"917.4138\"]," +
"[\"2012-05-06\",\"592.4823\"]]}").getBytes());
}
@Override
public void writeTo(OutputStream outputStream) throws IOException {
}
@Override
public boolean isStreaming() {
return false;
}
@Override
public void consumeContent() throws IOException {
}
}

304
lens/pom.xml Normal file
View file

@ -0,0 +1,304 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-lens</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Lens interpreter</name>
<url>http://www.apache.org</url>
<properties>
<lens.version>2.2.0-beta-incubating</lens.version>
<spring-shell.version>1.1.0.RELEASE</spring-shell.version>
<hadoop-common.version>2.4.0</hadoop-common.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>org.checkerframework</groupId>
<artifactId>jdk7</artifactId>
<version>1.9.1</version>
</dependency>
<dependency>
<groupId>org.apache.lens</groupId>
<artifactId>lens-cli</artifactId>
<version>${lens.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lens</groupId>
<artifactId>lens-client</artifactId>
<version>${lens.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>1.9.13</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>1.9.13</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
<version>1.9.11</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
<version>1.9.11</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-server</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>org.springframework.shell</groupId>
<artifactId>spring-shell</artifactId>
<version>${spring-shell.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop-common.version}</version>
<exclusions>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/lens</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/lens</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<filesets>
<fileset>
<directory>${basedir}/../interpreter/lens</directory>
<followSymlinks>false</followSymlinks>
</fileset>
</filesets>
</configuration>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>inmobi.repo</id>
<url>https://github.com/InMobi/mvn-repo/raw/master/releases</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>inmobi.snapshots</id>
<url>https://github.com/InMobi/mvn-repo/raw/master/snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>central</id>
<url>http://repo1.maven.org/maven2</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
<releases>
<enabled>true</enabled>
<updatePolicy>never</updatePolicy>
</releases>
<snapshots>
<enabled>false</enabled>
<updatePolicy>never</updatePolicy>
</snapshots>
</repository>
<repository>
<id>Codehaus repository</id>
<url>http://repository.codehaus.org/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>apache.snapshots.repo</id>
<url>https://repository.apache.org/content/groups/snapshots</url>
<name>Apache Snapshots Repository</name>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>default</id>
<url>https://repository.apache.org/content/groups/public/</url>
</repository>
<repository>
<id>projectlombok.org</id>
<url>http://projectlombok.org/mavenrepo</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<!-- see https://jira.springsource.org/browse/SHL-52 -->
<repository>
<id>ext-release-local</id>
<url>http://repo.springsource.org/simple/ext-release-local/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
</project>

44
lens/src/main/java/org/apache/zeppelin/lens/ExecutionDetail.java Normal file
View file

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
import org.apache.lens.client.LensClient;
import org.springframework.shell.core.JLineShell;
/**
* Pojo tracking query execution details
* Used to cancel the query
*/
public class ExecutionDetail {
private String queryHandle;
private LensClient lensClient;
private JLineShell shell;
ExecutionDetail(String qh, LensClient lensClient, JLineShell shell) {
this.queryHandle = qh;
this.lensClient = lensClient;
this.shell = shell;
}
public JLineShell getShell() {
return shell;
}
public String getQueryHandle() {
return queryHandle;
}
public LensClient getLensClient() {
return lensClient;
}
}

41
lens/src/main/java/org/apache/zeppelin/lens/LensBootstrap.java Normal file
View file

@ -0,0 +1,41 @@
/*
* Copyright 2011-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
import java.util.Properties;
import org.springframework.beans.factory.support.DefaultListableBeanFactory;
import org.springframework.beans.factory.support.RootBeanDefinition;
import org.springframework.context.support.GenericApplicationContext;
import org.springframework.shell.core.Shell;
/**
* workaround for https://github.com/spring-projects/spring-shell/issues/73
*/
public class LensBootstrap extends org.springframework.shell.Bootstrap {
public LensBootstrap() {
super();
}
public LensJLineShellComponent getLensJLineShellComponent() {
GenericApplicationContext ctx = (GenericApplicationContext) getApplicationContext();
RootBeanDefinition rbd = new RootBeanDefinition();
rbd.setBeanClass(LensJLineShellComponent.class);
DefaultListableBeanFactory bf = (DefaultListableBeanFactory) ctx.getBeanFactory();
bf.registerBeanDefinition(LensJLineShellComponent.class.getSimpleName(), rbd);
return ctx.getBean(LensJLineShellComponent.class);
}
}

451
lens/src/main/java/org/apache/zeppelin/lens/LensInterpreter.java Normal file
View file

@ -0,0 +1,451 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
import java.util.List;
import java.util.Properties;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.Map;
import java.util.LinkedHashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.io.ByteArrayOutputStream;
import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientConfig;
import org.apache.lens.client.LensClientSingletonWrapper;
import org.apache.lens.cli.commands.BaseLensCommand;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.shell.Bootstrap;
import org.springframework.shell.core.CommandResult;
import org.springframework.shell.core.JLineShell;
import org.springframework.shell.core.JLineShellComponent;
import org.springframework.shell.support.logging.HandlerUtils;
/**
* Lens interpreter for Zeppelin.
*/
public class LensInterpreter extends Interpreter {
static final Logger s_logger = LoggerFactory.getLogger(LensInterpreter.class);
static final String LENS_CLIENT_DBNAME = "lens.client.dbname";
static final String LENS_SERVER_URL = "lens.server.base.url";
static final String LENS_SESSION_CLUSTER_USER = "lens.session.cluster.user";
static final String LENS_PERSIST_RESULTSET = "lens.query.enable.persistent.resultset";
static final String ZEPPELIN_LENS_RUN_CONCURRENT_SESSION = "zeppelin.lens.run.concurrent";
static final String ZEPPELIN_LENS_CONCURRENT_SESSIONS = "zeppelin.lens.maxThreads";
static final String ZEPPELIN_MAX_ROWS = "zeppelin.lens.maxResults";
static final Map<String, Pattern> LENS_TABLE_FORMAT_REGEX = new LinkedHashMap<String, Pattern>() {
{
put("cubes", Pattern.compile(".*show\\s+cube.*"));
put("nativetables", Pattern.compile(".*show\\s+nativetable.*"));
put("storages", Pattern.compile(".*show\\s+storage.*"));
put("facts", Pattern.compile(".*show\\s+fact.*"));
put("dimensions", Pattern.compile(".*show\\s+dimension.*"));
put("params", Pattern.compile(".*show\\s+param.*"));
put("databases", Pattern.compile(".*show\\s+database.*"));
put("query results", Pattern.compile(".*query\\s+results.*"));
}
};
private static Pattern s_queryExecutePattern = Pattern.compile(".*query\\s+execute\\s+(.*)");
private static Map<String, ExecutionDetail> s_paraToQH =
new ConcurrentHashMap<String, ExecutionDetail> (); //tracks paragraphID -> Lens QueryHandle
private static Map<LensClient, Boolean> s_clientMap =
new ConcurrentHashMap<LensClient, Boolean>();
private int m_maxResults;
private int m_maxThreads;
private JLineShell m_shell;
private LensClientConfig m_lensConf;
private Bootstrap m_bs;
private LensClient m_lensClient;
static {
Interpreter.register(
"lens",
"lens",
LensInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(ZEPPELIN_LENS_RUN_CONCURRENT_SESSION, "true", "Run concurrent Lens Sessions")
.add(ZEPPELIN_LENS_CONCURRENT_SESSIONS, "10",
"If concurrency is true then how many threads?")
.add(ZEPPELIN_MAX_ROWS, "1000", "max number of rows to display")
.add(LENS_SERVER_URL, "http://<hostname>:<port>/lensapi", "The URL for Lens Server")
.add(LENS_CLIENT_DBNAME, "default", "The database schema name")
.add(LENS_PERSIST_RESULTSET, "false", "Apache Lens to persist result in HDFS?")
.add(LENS_SESSION_CLUSTER_USER, "default", "Hadoop cluster username").build());
}
public LensInterpreter(Properties property) {
super(property);
try {
m_lensConf = new LensClientConfig();
m_lensConf.set(LENS_SERVER_URL, property.get(LENS_SERVER_URL).toString());
m_lensConf.set(LENS_CLIENT_DBNAME, property.get(LENS_CLIENT_DBNAME).toString());
m_lensConf.set(LENS_SESSION_CLUSTER_USER, property.get(LENS_SESSION_CLUSTER_USER).toString());
m_lensConf.set(LENS_PERSIST_RESULTSET, property.get(LENS_PERSIST_RESULTSET).toString());
try {
m_maxResults = Integer.parseInt(property.get(ZEPPELIN_MAX_ROWS).toString());
} catch (NumberFormatException|NullPointerException e) {
m_maxResults = 1000;
s_logger.error("unable to parse " + ZEPPELIN_MAX_ROWS + " :"
+ property.get(ZEPPELIN_MAX_ROWS), e);
}
try {
m_maxThreads = Integer.parseInt(property.get(ZEPPELIN_LENS_CONCURRENT_SESSIONS).toString());
} catch (NumberFormatException|NullPointerException e) {
m_maxThreads = 10;
s_logger.error("unable to parse " + ZEPPELIN_LENS_CONCURRENT_SESSIONS + " :"
+ property.get(ZEPPELIN_LENS_CONCURRENT_SESSIONS), e);
}
s_logger.info("LensInterpreter created");
}
catch (Exception e) {
e.printStackTrace();
s_logger.error("unable to create lens interpreter", e);
}
}
private Bootstrap createBootstrap() {
return new LensBootstrap();
}
private JLineShell getJLineShell(Bootstrap bs) {
if (bs instanceof LensBootstrap) {
return ((LensBootstrap) bs).getLensJLineShellComponent();
} else {
return bs.getJLineShellComponent();
}
}
protected void init() {
try {
m_bs = createBootstrap();
m_shell = getJLineShell(m_bs);
} catch (Exception ex) {
s_logger.error("could not initialize commandLine", ex);
}
}
@Override
public void open() {
s_logger.info("LensInterpreter opening");
m_lensClient = new LensClient(m_lensConf);
LensClientSingletonWrapper.instance().setClient(m_lensClient);
init();
s_logger.info("LensInterpreter opened");
}
@Override
public void close() {
closeConnections();
s_logger.info("LensInterpreter closed");
}
private static void closeConnections() {
for (LensClient cl : s_clientMap.keySet()) {
if (cl.isConnectionOpen()) {
closeLensClient(cl);
}
}
}
private static void closeLensClient(LensClient lensClient) {
try {
lensClient.closeConnection();
} catch (Exception e) {
s_logger.error("unable to close lensClient", e);
}
}
private LensClient createAndSetLensClient(Bootstrap bs) {
LensClient lensClient = null;
try {
lensClient = new LensClient(m_lensConf);
for (String beanName : bs.getApplicationContext().getBeanDefinitionNames()) {
if (bs.getApplicationContext().getBean(beanName) instanceof BaseLensCommand) {
((BaseLensCommand) bs.getApplicationContext().getBean(beanName))
.setClient(lensClient);
}
}
} catch (Exception e) {
s_logger.error("unable to create lens client", e);
throw e;
}
return lensClient;
}
private InterpreterResult HandleHelp(JLineShell shell, String st) {
java.util.logging.StreamHandler sh = null;
java.util.logging.Logger springLogger = null;
java.util.logging.Formatter formatter = new java.util.logging.Formatter() {
public String format(java.util.logging.LogRecord record) {
return record.getMessage();
}
};
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
sh = new java.util.logging.StreamHandler(baos, formatter);
springLogger = HandlerUtils.getLogger(org.springframework.shell.core.SimpleParser.class);
springLogger.addHandler(sh);
shell.executeCommand(st);
} catch (Exception e) {
s_logger.error(e.getMessage(), e);
return new InterpreterResult(Code.ERROR, e.getMessage());
}
finally {
sh.flush();
springLogger.removeHandler(sh);
sh.close();
}
return new InterpreterResult(Code.SUCCESS, baos.toString());
}
private String modifyQueryStatement(String st) {
Matcher matcher = s_queryExecutePattern.matcher(st.toLowerCase());
if (!matcher.find()) {
return st;
}
StringBuilder sb = new StringBuilder("query execute ");
if (!st.toLowerCase().matches(".*--async\\s+true")) {
sb.append("--async true ");
}
sb.append(matcher.group(1));
if (!st.toLowerCase().matches(".*limit\\s+\\d+.*")) {
sb.append(" limit ");
sb.append(m_maxResults);
}
return sb.toString();
}
@Override
public InterpreterResult interpret(String input, InterpreterContext context) {
if (input == null || input.length() == 0) {
return new InterpreterResult(Code.ERROR, "no command submitted");
}
String st = input.replaceAll("\\n", " ");
s_logger.info("LensInterpreter command: " + st);
Bootstrap bs = createBootstrap();
JLineShell shell = getJLineShell(bs);
CommandResult res = null;
LensClient lensClient = null;
String qh = null;
if (st.trim().startsWith("help")) {
return HandleHelp(shell, st);
}
try {
lensClient = createAndSetLensClient(bs);
s_clientMap.put(lensClient, true);
String lensCommand = modifyQueryStatement(st);
s_logger.info("executing command : " + lensCommand);
res = shell.executeCommand(lensCommand);
if (!lensCommand.equals(st) && res != null
&& res.getResult() != null
&& res.getResult().toString().trim().matches("[a-z0-9-]+")) {
// setup query progress tracking
qh = res.getResult().toString();
s_paraToQH.put(context.getParagraphId(),
new ExecutionDetail(qh, lensClient, shell));
String getResultsCmd = "query results --async false " + qh;
s_logger.info("executing query results command : " + context.getParagraphId()
+ " : " + getResultsCmd);
res = shell.executeCommand(getResultsCmd);
s_paraToQH.remove(context.getParagraphId());
}
} catch (Exception ex) {
s_logger.error("error in interpret", ex);
return new InterpreterResult(Code.ERROR, ex.getMessage());
}
finally {
if (shell != null) {
closeShell(shell);
}
if (lensClient != null) {
closeLensClient(lensClient);
s_clientMap.remove(lensClient);
}
if (qh != null) {
s_paraToQH.remove(context.getParagraphId());
}
}
return new InterpreterResult(Code.SUCCESS, formatResult(st, res));
}
private void closeShell(JLineShell shell) {
if (shell instanceof LensJLineShellComponent) {
((LensJLineShellComponent) shell).stop();
} else {
((JLineShellComponent) shell).stop();
}
}
private String formatResult(String st, CommandResult result) {
if (result == null) {
return "error in interpret, no result object returned";
}
if (!result.isSuccess() || result.getResult() == null) {
if (result.getException() != null) {
return result.getException().getMessage();
//try describe cube (without cube name)- error is written as a warning,
//but not returned to result object
} else {
return "error in interpret, unable to execute command";
}
}
StringBuilder sb = new StringBuilder();
for (Map.Entry<String, Pattern> entry : LENS_TABLE_FORMAT_REGEX.entrySet()) {
if (entry.getValue().matcher(st.toLowerCase()).find()) {
sb.append("%table " + entry.getKey() + " \n");
break;
}
}
if (s_queryExecutePattern.matcher(st.toLowerCase()).find() &&
result.getResult().toString().contains(" rows process in (")) {
sb.append("%table ");
}
if (sb.length() > 0) {
return sb.append(result.getResult().toString()).toString();
}
return result.getResult().toString();
//Lens sends error messages without setting result.isSuccess() = false.
}
@Override
public void cancel(InterpreterContext context) {
if (!s_paraToQH.containsKey(context.getParagraphId())) {
s_logger.error("ignoring cancel from " + context.getParagraphId());
return;
}
String qh = s_paraToQH.get(context.getParagraphId()).getQueryHandle();
s_logger.info("preparing to cancel : (" + context.getParagraphId() + ") :" + qh);
Bootstrap bs = createBootstrap();
JLineShell shell = getJLineShell(bs);
LensClient lensClient = null;
try {
lensClient = createAndSetLensClient(bs);
s_clientMap.put(lensClient, true);
s_logger.info("invoke query kill (" + context.getParagraphId() + ") " + qh);
CommandResult res = shell.executeCommand("query kill " + qh);
s_logger.info("query kill returned (" + context.getParagraphId() + ") " + qh
+ " with: " + res.getResult());
} catch (Exception e) {
s_logger.error("unable to kill query ("
+ context.getParagraphId() + ") " + qh, e);
} finally {
try {
if (lensClient != null) {
closeLensClient(lensClient);
s_clientMap.remove(lensClient);
}
closeLensClient(s_paraToQH.get(context.getParagraphId()).getLensClient());
closeShell(s_paraToQH.get(context.getParagraphId()).getShell());
} catch (Exception e) {
// ignore
}
s_paraToQH.remove(context.getParagraphId());
closeShell(shell);
}
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
if (s_paraToQH.containsKey(context.getParagraphId())) {
s_logger.info("number of items for which progress can be reported :" + s_paraToQH.size());
s_logger.info("number of open lensclient :" + s_clientMap.size());
Bootstrap bs = createBootstrap();
JLineShell shell = getJLineShell(bs);
LensClient lensClient = null;
String qh = s_paraToQH.get(context.getParagraphId()).getQueryHandle();
try {
s_logger.info("fetch query status for : (" + context.getParagraphId() + ") :" + qh);
lensClient = createAndSetLensClient(bs);
s_clientMap.put(lensClient, true);
CommandResult res = shell.executeCommand("query status " + qh);
s_logger.info(context.getParagraphId() + " --> " + res.getResult().toString());
//change to debug
Pattern pattern = Pattern.compile(".*(Progress : (\\d\\.\\d)).*");
Matcher matcher = pattern.matcher(res.getResult().toString().replaceAll("\\n", " "));
if (matcher.find(2)) {
Double d = Double.parseDouble(matcher.group(2)) * 100;
if (d.intValue() == 100) {
s_paraToQH.remove(context.getParagraphId());
}
return d.intValue();
} else {
return 1;
}
}
catch (Exception e) {
s_logger.error("unable to get progress for (" + context.getParagraphId() + ") :" + qh, e);
s_paraToQH.remove(context.getParagraphId());
return 0;
} finally {
if (lensClient != null) {
closeLensClient(lensClient);
s_clientMap.remove(lensClient);
}
if (shell != null) {
closeShell(shell);
}
}
}
return 0;
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
public boolean concurrentRequests() {
return Boolean.parseBoolean(getProperty(ZEPPELIN_LENS_RUN_CONCURRENT_SESSION));
}
@Override
public Scheduler getScheduler() {
if (concurrentRequests()) {
return SchedulerFactory.singleton().createOrGetParallelScheduler(
LensInterpreter.class.getName() + this.hashCode(), m_maxThreads);
} else {
return super.getScheduler();
}
}
}

244
lens/src/main/java/org/apache/zeppelin/lens/LensJLineShellComponent.java Normal file
View file

@ -0,0 +1,244 @@
/*
* Copyright 2011-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
import java.util.Map;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactoryUtils;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.SmartLifecycle;
import org.springframework.shell.CommandLine;
import org.springframework.shell.plugin.BannerProvider;
import org.springframework.shell.plugin.HistoryFileNameProvider;
import org.springframework.shell.plugin.PluginUtils;
import org.springframework.shell.plugin.PromptProvider;
import org.springframework.shell.core.*;
/**
* workaround for https://github.com/spring-projects/spring-shell/issues/73
*/
public class LensJLineShellComponent extends JLineShell
implements SmartLifecycle, ApplicationContextAware, InitializingBean {
@Autowired
private CommandLine commandLine;
private volatile boolean running = false;
private Thread shellThread;
private ApplicationContext applicationContext;
private boolean printBanner = true;
private String historyFileName;
private String promptText;
private String productName;
private String banner;
private String version;
private String welcomeMessage;
private ExecutionStrategy executionStrategy = new LensSimpleExecutionStrategy();
private SimpleParser parser = new SimpleParser();
public SimpleParser getSimpleParser() {
return parser;
}
public boolean isAutoStartup() {
return false;
}
public void stop(Runnable callback) {
stop();
callback.run();
}
public int getPhase() {
return 1;
}
public void start() {
//customizePlug must run before start thread to take plugin's configuration into effect
customizePlugin();
shellThread = new Thread(this, "Spring Shell");
shellThread.start();
running = true;
}
public void stop() {
if (running) {
closeShell();
running = false;
}
}
public boolean isRunning() {
return running;
}
@SuppressWarnings("rawtypes")
public void afterPropertiesSet() {
Map<String, CommandMarker> commands =
BeanFactoryUtils.beansOfTypeIncludingAncestors(applicationContext,
CommandMarker.class);
for (CommandMarker command : commands.values()) {
getSimpleParser().add(command);
}
Map<String, Converter> converters = BeanFactoryUtils
.beansOfTypeIncludingAncestors(applicationContext, Converter.class);
for (Converter<?> converter : converters.values()) {
getSimpleParser().add(converter);
}
setHistorySize(commandLine.getHistorySize());
if (commandLine.getShellCommandsToExecute() != null) {
setPrintBanner(false);
}
}
/**
* wait the shell command to complete by typing "quit" or "exit"
*
*/
public void waitForComplete() {
try {
shellThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
@Override
protected ExecutionStrategy getExecutionStrategy() {
return executionStrategy;
}
@Override
protected Parser getParser() {
return parser;
}
@Override
public String getStartupNotifications() {
return null;
}
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
public void customizePlugin() {
this.historyFileName = getHistoryFileName();
this.promptText = getPromptText();
String[] banner = getBannerText();
this.banner = banner[0];
this.welcomeMessage = banner[1];
this.version = banner[2];
this.productName = banner[3];
}
/**
* get history file name from provider. The provider has highest order
* <link>org.springframework.core.Ordered.getOder</link> will win.
*
* @return history file name
*/
protected String getHistoryFileName() {
HistoryFileNameProvider historyFileNameProvider = PluginUtils
.getHighestPriorityProvider(this.applicationContext, HistoryFileNameProvider.class);
String providerHistoryFileName = historyFileNameProvider.getHistoryFileName();
if (providerHistoryFileName != null) {
return providerHistoryFileName;
} else {
return historyFileName;
}
}
/**
* get prompt text from provider. The provider has highest order
* <link>org.springframework.core.Ordered.getOder</link> will win.
*
* @return prompt text
*/
protected String getPromptText() {
PromptProvider promptProvider = PluginUtils
.getHighestPriorityProvider(this.applicationContext, PromptProvider.class);
String providerPromptText = promptProvider.getPrompt();
if (providerPromptText != null) {
return providerPromptText;
} else {
return promptText;
}
}
/**
* Get Banner and Welcome Message from provider. The provider has highest order
* <link>org.springframework.core.Ordered.getOder</link> will win.
* @return BannerText[0]: Banner
* BannerText[1]: Welcome Message
* BannerText[2]: Version
* BannerText[3]: Product Name
*/
private String[] getBannerText() {
String[] bannerText = new String[4];
BannerProvider provider = PluginUtils
.getHighestPriorityProvider(this.applicationContext, BannerProvider.class);
bannerText[0] = provider.getBanner();
bannerText[1] = provider.getWelcomeMessage();
bannerText[2] = provider.getVersion();
bannerText[3] = provider.getProviderName();
return bannerText;
}
public void printBannerAndWelcome() {
if (printBanner) {
logger.info(this.banner);
logger.info(getWelcomeMessage());
}
}
/**
* get the welcome message at start.
*
* @return welcome message
*/
public String getWelcomeMessage() {
return this.welcomeMessage;
}
/**
* @param printBanner the printBanner to set
*/
public void setPrintBanner(boolean printBanner) {
this.printBanner = printBanner;
}
protected String getProductName() {
return productName;
}
protected String getVersion() {
return version;
}
}

86
lens/src/main/java/org/apache/zeppelin/lens/LensSimpleExecutionStrategy.java Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright 2011-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
import org.springframework.shell.core.*;
import java.util.logging.Logger;
import org.springframework.shell.event.ParseResult;
import org.springframework.shell.support.logging.HandlerUtils;
import org.springframework.util.Assert;
import org.springframework.util.ReflectionUtils;
/**
* workaround for https://github.com/spring-projects/spring-shell/issues/73
*/
public class LensSimpleExecutionStrategy implements ExecutionStrategy {
private static final Logger logger = HandlerUtils.getLogger(LensSimpleExecutionStrategy.class);
public Object execute(ParseResult parseResult) throws RuntimeException {
Assert.notNull(parseResult, "Parse result required");
logger.info("LensSimpleExecutionStrategy execute method invoked");
synchronized (this) {
Assert.isTrue(isReadyForCommands(), "SimpleExecutionStrategy not yet ready for commands");
Object target = parseResult.getInstance();
if (target instanceof ExecutionProcessor) {
ExecutionProcessor processor = ((ExecutionProcessor) target);
parseResult = processor.beforeInvocation(parseResult);
try {
Object result = invoke(parseResult);
processor.afterReturningInvocation(parseResult, result);
return result;
} catch (Throwable th) {
processor.afterThrowingInvocation(parseResult, th);
return handleThrowable(th);
}
}
else {
return invoke(parseResult);
}
}
}
private Object invoke(ParseResult parseResult) {
try {
return ReflectionUtils.invokeMethod(parseResult.getMethod(),
parseResult.getInstance(), parseResult.getArguments());
} catch (Throwable th) {
logger.severe("Command failed " + th);
return handleThrowable(th);
}
}
private Object handleThrowable(Throwable th) {
if (th instanceof Error) {
throw ((Error) th);
}
if (th instanceof RuntimeException) {
throw ((RuntimeException) th);
}
throw new RuntimeException(th);
}
public boolean isReadyForCommands() {
return true;
}
public void terminate() {
// do nothing
}
}

71
lens/src/test/java/org/apache/zeppelin/lens/LensInterpreterTest.java Normal file
View file

@ -0,0 +1,71 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
import static org.junit.Assert.assertEquals;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.apache.zeppelin.lens.LensInterpreter.*;
/**
* Lens interpreter unit tests
*/
public class LensInterpreterTest {
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@Test
public void test() {
Properties prop = new Properties();
prop.setProperty(LENS_SERVER_URL, "http://127.0.0.1:9999/lensapi");
prop.setProperty(LENS_CLIENT_DBNAME, "default");
prop.setProperty(LENS_PERSIST_RESULTSET, "false");
prop.setProperty(LENS_SESSION_CLUSTER_USER, "default");
prop.setProperty(ZEPPELIN_MAX_ROWS, "1000");
prop.setProperty(ZEPPELIN_LENS_RUN_CONCURRENT_SESSION, "true");
prop.setProperty(ZEPPELIN_LENS_CONCURRENT_SESSIONS, "10");
LensInterpreter t = new MockLensInterpreter(prop);
t.open();
//simple help test
InterpreterResult result = t.interpret("help", null);
assertEquals(result.type(), InterpreterResult.Type.TEXT);
//assertEquals("unable to find 'query execute' in help message",
// result.message().contains("query execute"), result.message());
t.close();
}
class MockLensInterpreter extends LensInterpreter {
public MockLensInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
super.init();
}
}
}

5
markdown/pom.xml
View file

@ -22,13 +22,14 @@
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.5.0-incubating-SNAPSHOT</version>
<version>0.6.0-incubating-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-markdown</artifactId>
<packaging>jar</packaging>
<version>0.5.0-incubating-SNAPSHOT</version>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Markdown interpreter</name>
<url>http://zeppelin.incubator.apache.org</url>

99
notebook/2A94M5J1Z/note.json
View file

@ -13,7 +13,7 @@
"groups": [],
"scatter": {}
},
"editorHide": false
"editorHide": true
},
"settings": {
"params": {},
@ -32,42 +32,9 @@
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"title": "Prepare data",
"text": "import sys.process._\n//you will need \u0027wget\u0027 tool to download\n\"wget http://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank.zip\" !\n\"mkdir data\" !\n\"unzip bank.zip -d data\" !\n\"rm bank.zip\" !",
"config": {
"colWidth": 12.0,
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false,
"keys": [],
"values": [],
"groups": [],
"scatter": {}
},
"title": true
},
"settings": {
"params": {},
"forms": {}
},
"jobName": "paragraph_1417656535623_-196593192",
"id": "20141204-102855_1590713432",
"result": {
"code": "SUCCESS",
"type": "TEXT",
"msg": "import sys.process._\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres1: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres2: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres3: Int \u003d 0\nwarning: there were 1 feature warning(s); re-run with -feature for details\nres4: Int \u003d 0\n"
},
"dateCreated": "Dec 4, 2014 10:28:55 AM",
"dateStarted": "Apr 1, 2015 9:11:12 PM",
"dateFinished": "Apr 1, 2015 9:11:22 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"title": "Load data into table",
"text": "import sys.process._\n// sc is an existing SparkContext.\nval sqlContext \u003d new org.apache.spark.sql.SQLContext(sc)\n\n\nval zeppelinHome \u003d (\"pwd\" !!).replace(\"\\n\", \"\")\nval bankText \u003d sc.textFile(s\"file://$zeppelinHome/data/bank-full.csv\")\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")\n\n",
"text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")",
"config": {
"colWidth": 12.0,
"graph": {
@ -90,11 +57,11 @@
"result": {
"code": "SUCCESS",
"type": "TEXT",
"msg": "import sys.process._\nsqlContext: org.apache.spark.sql.SQLContext \u003d org.apache.spark.sql.SQLContext@2c91e2d6\nwarning: there were 1 feature warning(s); re-run with -feature for details\nzeppelinHome: String \u003d /home/langley/lab/incubator-zeppelin\nbankText: org.apache.spark.rdd.RDD[String] \u003d /home/langley/lab/incubator-zeppelin/data/bank-full.csv MapPartitionsRDD[1] at textFile at \u003cconsole\u003e:31\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n"
"msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n"
},
"dateCreated": "Feb 10, 2015 1:52:59 AM",
"dateStarted": "Apr 1, 2015 9:11:28 PM",
"dateFinished": "Apr 1, 2015 9:11:39 PM",
"dateStarted": "Jul 3, 2015 1:43:40 PM",
"dateFinished": "Jul 3, 2015 1:43:45 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
@ -144,11 +111,11 @@
"result": {
"code": "SUCCESS",
"type": "TABLE",
"msg": "age\tvalue\n18\t12\n19\t35\n20\t50\n21\t79\n22\t129\n23\t202\n24\t302\n25\t527\n26\t805\n27\t909\n28\t1038\n29\t1185\n"
"msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n"
},
"dateCreated": "Feb 10, 2015 1:53:02 AM",
"dateStarted": "Apr 1, 2015 9:11:43 PM",
"dateFinished": "Apr 1, 2015 9:11:45 PM",
"dateStarted": "Jul 3, 2015 1:43:17 PM",
"dateFinished": "Jul 3, 2015 1:43:23 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
@ -206,11 +173,11 @@
"result": {
"code": "SUCCESS",
"type": "TABLE",
"msg": "age\tvalue\n18\t12\n19\t35\n20\t50\n21\t79\n22\t129\n23\t202\n24\t302\n25\t527\n26\t805\n27\t909\n28\t1038\n29\t1185\n30\t1757\n31\t1996\n32\t2085\n33\t1972\n34\t1930\n"
"msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n"
},
"dateCreated": "Feb 12, 2015 2:54:04 PM",
"dateStarted": "Apr 1, 2015 9:12:03 PM",
"dateFinished": "Apr 1, 2015 9:12:03 PM",
"dateStarted": "Jul 3, 2015 1:43:28 PM",
"dateFinished": "Jul 3, 2015 1:43:29 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
@ -279,11 +246,11 @@
"result": {
"code": "SUCCESS",
"type": "TABLE",
"msg": "age\tvalue\n18\t12\n19\t35\n20\t47\n21\t74\n22\t120\n23\t175\n24\t248\n25\t423\n26\t615\n27\t658\n28\t697\n29\t683\n30\t1012\n31\t1017\n32\t941\n33\t746\n34\t650\n35\t631\n36\t538\n37\t453\n38\t394\n39\t346\n40\t257\n41\t241\n42\t218\n43\t183\n44\t170\n45\t146\n46\t130\n47\t100\n48\t124\n49\t101\n50\t76\n51\t72\n52\t62\n53\t71\n54\t55\n55\t54\n56\t45\n57\t38\n58\t35\n59\t36\n60\t27\n61\t5\n63\t2\n66\t5\n67\t3\n68\t4\n69\t2\n70\t1\n71\t1\n72\t5\n73\t2\n77\t1\n83\t2\n86\t1\n"
"msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n"
},
"dateCreated": "Feb 13, 2015 11:04:22 PM",
"dateStarted": "Apr 1, 2015 9:12:10 PM",
"dateFinished": "Apr 1, 2015 9:12:10 PM",
"dateStarted": "Jul 3, 2015 1:43:33 PM",
"dateFinished": "Jul 3, 2015 1:43:34 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
@ -299,7 +266,8 @@
"values": [],
"groups": [],
"scatter": {}
}
},
"editorHide": true
},
"settings": {
"params": {},
@ -319,20 +287,53 @@
"progressUpdateIntervalMs": 500
},
{
"config": {},
"text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```",
"config": {
"colWidth": 12.0,
"graph": {
"mode": "table",
"height": 300.0,
"optionOpen": false,
"keys": [],
"values": [],
"groups": [],
"scatter": {}
},
"editorHide": true
},
"settings": {
"params": {},
"forms": {}
},
"jobName": "paragraph_1427420818407_872443482",
"id": "20150326-214658_12335843",
"result": {
"code": "SUCCESS",
"type": "HTML",
"msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"
},
"dateCreated": "Mar 26, 2015 9:46:58 PM",
"dateStarted": "Jul 3, 2015 1:44:56 PM",
"dateFinished": "Jul 3, 2015 1:44:56 PM",
"status": "FINISHED",
"progressUpdateIntervalMs": 500
},
{
"config": {},
"settings": {
"params": {},
"forms": {}
},
"jobName": "paragraph_1435955447812_-158639899",
"id": "20150703-133047_853701097",
"dateCreated": "Jul 3, 2015 1:30:47 PM",
"status": "READY",
"progressUpdateIntervalMs": 500
}
],
"name": "Zeppelin Tutorial",
"id": "2A94M5J1Z",
"angularObjects": {},
"config": {
"looknfeel": "default"
},

143
phoenix/pom.xml Normal file
View file

@ -0,0 +1,143 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-phoenix</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: Apache Phoenix Interpreter</name>
<description>Zeppelin interprter for Apache Phoenix</description>
<url>http://zeppelin.incubator.apache.org</url>
<properties>
<phoenix.version>4.4.0-HBase-1.0</phoenix.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>
<version>${phoenix.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.mockrunner</groupId>
<artifactId>mockrunner-jdbc</artifactId>
<version>1.0.8</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/phoenix</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/phoenix</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

239
phoenix/src/main/java/org/apache/zeppelin/phoenix/PhoenixInterpreter.java Normal file
View file

@ -0,0 +1,239 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.phoenix;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Phoenix interpreter for Zeppelin.
*/
public class PhoenixInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(PhoenixInterpreter.class);
private static final String EXPLAIN_PREDICATE = "EXPLAIN ";
private static final String UPDATE_HEADER = "UPDATES ";
private static final String WS = " ";
private static final String NEWLINE = "\n";
private static final String TAB = "\t";
private static final String TABLE_MAGIC_TAG = "%table ";
static final String PHOENIX_JDBC_URL = "phoenix.jdbc.url";
static final String PHOENIX_JDBC_USER = "phoenix.user";
static final String PHOENIX_JDBC_PASSWORD = "phoenix.password";
static final String PHOENIX_MAX_RESULT = "phoenix.max.result";
static final String PHOENIX_JDBC_DRIVER_NAME = "phoenix.driver.name";
static final String DEFAULT_JDBC_URL = "jdbc:phoenix:localhost:2181:/hbase-unsecure";
static final String DEFAULT_JDBC_USER = "";
static final String DEFAULT_JDBC_PASSWORD = "";
static final String DEFAULT_MAX_RESULT = "1000";
static final String DEFAULT_JDBC_DRIVER_NAME = "org.apache.phoenix.jdbc.PhoenixDriver";
private Connection jdbcConnection;
private Statement currentStatement;
private Exception exceptionOnConnect;
private int maxResult;
static {
Interpreter.register(
"sql",
"phoenix",
PhoenixInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(PHOENIX_JDBC_URL, DEFAULT_JDBC_URL, "Phoenix JDBC connection string")
.add(PHOENIX_JDBC_USER, DEFAULT_JDBC_USER, "The Phoenix user")
.add(PHOENIX_JDBC_PASSWORD, DEFAULT_JDBC_PASSWORD, "The password for the Phoenix user")
.add(PHOENIX_MAX_RESULT, DEFAULT_MAX_RESULT, "Max number of SQL results to display.")
.add(PHOENIX_JDBC_DRIVER_NAME, DEFAULT_JDBC_DRIVER_NAME, "Phoenix Driver classname.")
.build()
);
}
public PhoenixInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
logger.info("Jdbc open connection called!");
close();
try {
Class.forName(getProperty(PHOENIX_JDBC_DRIVER_NAME));
maxResult = Integer.valueOf(getProperty(PHOENIX_MAX_RESULT));
jdbcConnection = DriverManager.getConnection(
getProperty(PHOENIX_JDBC_URL),
getProperty(PHOENIX_JDBC_USER),
getProperty(PHOENIX_JDBC_PASSWORD)
);
exceptionOnConnect = null;
logger.info("Successfully created Jdbc connection");
}
catch (ClassNotFoundException | SQLException e) {
logger.error("Cannot open connection", e);
exceptionOnConnect = e;
}
}
@Override
public void close() {
logger.info("Jdbc close connection called!");
try {
if (getJdbcConnection() != null) {
getJdbcConnection().close();
}
} catch (SQLException e) {
logger.error("Cannot close connection", e);
}
finally {
exceptionOnConnect = null;
}
}
private String clean(boolean isExplain, String str){
return (isExplain || str == null) ? str : str.replace(TAB, WS).replace(NEWLINE, WS);
}
private InterpreterResult executeSql(String sql) {
try {
if (exceptionOnConnect != null) {
return new InterpreterResult(Code.ERROR, exceptionOnConnect.getMessage());
}
currentStatement = getJdbcConnection().createStatement();
boolean isExplain = StringUtils.containsIgnoreCase(sql, EXPLAIN_PREDICATE);
StringBuilder msg = (isExplain) ? new StringBuilder() : new StringBuilder(TABLE_MAGIC_TAG);
ResultSet res = null;
try {
boolean hasResult = currentStatement.execute(sql);
if (hasResult){ //If query had results
res = currentStatement.getResultSet();
//Append column names
ResultSetMetaData md = res.getMetaData();
String row = clean(isExplain, md.getColumnName(1));
for (int i = 2; i < md.getColumnCount() + 1; i++)
row += TAB + clean(isExplain, md.getColumnName(i));
msg.append(row + NEWLINE);
//Append rows
int rowCount = 0;
while (res.next() && rowCount < getMaxResult()) {
row = clean(isExplain, res.getString(1));
for (int i = 2; i < md.getColumnCount() + 1; i++)
row += TAB + clean(isExplain, res.getString(i));
msg.append(row + NEWLINE);
rowCount++;
}
}
else { // May have been upsert or DDL
msg.append(UPDATE_HEADER + NEWLINE +
"Rows affected: " + currentStatement.getUpdateCount()
+ NEWLINE);
}
} finally {
try {
if (res != null) res.close();
getJdbcConnection().commit();
currentStatement.close();
} finally {
currentStatement = null;
}
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
}
catch (SQLException ex) {
logger.error("Can not run " + sql, ex);
return new InterpreterResult(Code.ERROR, ex.getMessage());
}
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.info("Run SQL command '" + cmd + "'");
return executeSql(cmd);
}
@Override
public void cancel(InterpreterContext context) {
if (currentStatement != null) {
try {
currentStatement.cancel();
}
catch (SQLException ex) {
}
finally {
currentStatement = null;
}
}
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
PhoenixInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
public Connection getJdbcConnection() {
return jdbcConnection;
}
public int getMaxResult() {
return maxResult;
}
}

226
phoenix/src/test/java/org/apache/zeppelin/phoenix/PhoenixInterpreterTest.java Normal file
View file

@ -0,0 +1,226 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.phoenix;
import static org.apache.zeppelin.phoenix.PhoenixInterpreter.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
import java.sql.SQLException;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Matchers;
import org.mockito.Mockito;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
import com.mockrunner.jdbc.StatementResultSetHandler;
import com.mockrunner.mock.jdbc.MockConnection;
import com.mockrunner.mock.jdbc.MockResultSet;
/**
* Phoenix interpreter unit tests
*/
public class PhoenixInterpreterTest extends BasicJDBCTestCaseAdapter {
private PhoenixInterpreter phoenixInterpreter = null;
private MockResultSet result = null;
@Before
public void beforeTest() {
MockConnection connection = getJDBCMockObjectFactory().getMockConnection();
StatementResultSetHandler statementHandler = connection.getStatementResultSetHandler();
result = statementHandler.createResultSet();
statementHandler.prepareGlobalResultSet(result);
Properties properties = new Properties();
properties.put(PHOENIX_JDBC_DRIVER_NAME, DEFAULT_JDBC_DRIVER_NAME);
properties.put(PHOENIX_JDBC_URL, DEFAULT_JDBC_URL);
properties.put(PHOENIX_JDBC_USER, DEFAULT_JDBC_USER);
properties.put(PHOENIX_JDBC_PASSWORD, DEFAULT_JDBC_PASSWORD);
properties.put(PHOENIX_MAX_RESULT, DEFAULT_MAX_RESULT);
phoenixInterpreter = spy(new PhoenixInterpreter(properties));
when(phoenixInterpreter.getJdbcConnection()).thenReturn(connection);
}
@Test
public void testOpenCommandIdempotency() throws SQLException {
// Ensure that an attempt to open new connection will clean any remaining connections
phoenixInterpreter.open();
phoenixInterpreter.open();
phoenixInterpreter.open();
verify(phoenixInterpreter, times(3)).open();
verify(phoenixInterpreter, times(3)).close();
}
@Test
public void testDefaultProperties() throws SQLException {
PhoenixInterpreter phoenixInterpreter = new PhoenixInterpreter(new Properties());
assertEquals(DEFAULT_JDBC_DRIVER_NAME,
phoenixInterpreter.getProperty(PHOENIX_JDBC_DRIVER_NAME));
assertEquals(DEFAULT_JDBC_URL, phoenixInterpreter.getProperty(PHOENIX_JDBC_URL));
assertEquals(DEFAULT_JDBC_USER, phoenixInterpreter.getProperty(PHOENIX_JDBC_USER));
assertEquals(DEFAULT_JDBC_PASSWORD,
phoenixInterpreter.getProperty(PHOENIX_JDBC_PASSWORD));
assertEquals(DEFAULT_MAX_RESULT, phoenixInterpreter.getProperty(PHOENIX_MAX_RESULT));
}
@Test
public void testConnectionClose() throws SQLException {
PhoenixInterpreter phoenixInterpreter = spy(new PhoenixInterpreter(new Properties()));
when(phoenixInterpreter.getJdbcConnection()).thenReturn(
getJDBCMockObjectFactory().getMockConnection());
phoenixInterpreter.close();
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
verifyConnectionClosed();
}
@Test
public void testStatementCancel() throws SQLException {
PhoenixInterpreter phoenixInterpreter = spy(new PhoenixInterpreter(new Properties()));
when(phoenixInterpreter.getJdbcConnection()).thenReturn(
getJDBCMockObjectFactory().getMockConnection());
phoenixInterpreter.cancel(null);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
assertFalse("Cancel operation should not close the connection", phoenixInterpreter
.getJdbcConnection().isClosed());
}
@Test
public void testSelectQuery() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
result.addColumn("col2", new String[] {"val21", "val22"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("col1\tcol2\nval11\tval21\nval12\tval22\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testSelectQueryMaxResult() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1);
String sqlQuery = "select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
result.addColumn("col2", new String[] {"val21", "val22"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("col1\tcol2\nval11\tval21\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testSelectQueryWithSpecialCharacters() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "select * from t";
result.addColumn("co\tl1", new String[] {"val11", "va\tl1\n2"});
result.addColumn("co\nl2", new String[] {"v\nal21", "val\t22"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("co l1\tco l2\nval11\tv al21\nva l1 2\tval 22\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testExplainQuery() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "explain select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TEXT, interpreterResult.type());
assertEquals("col1\nval11\nval12\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testExplainQueryWithSpecialCharachters() throws SQLException {
when(phoenixInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "explain select * from t";
result.addColumn("co\tl\n1", new String[] {"va\nl11", "va\tl\n12"});
InterpreterResult interpreterResult = phoenixInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TEXT, interpreterResult.type());
assertEquals("co\tl\n1\nva\nl11\nva\tl\n12\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
}

789
pom.xml Normal file → Executable file
View file

@ -35,15 +35,15 @@
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin</artifactId>
<packaging>pom</packaging>
<version>0.5.0-incubating-SNAPSHOT</version>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin</name>
<description>Zeppelin project</description>
<url>http://zeppelin.incubator.apache.org/</url>
<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>7</version>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>17</version>
</parent>
<licenses>
@ -55,9 +55,9 @@
</licenses>
<scm>
<url>http://zeppelin.incubator.apache.org</url>
<connection>scm:git:git:git@github.com:apache/incubator-zeppelin.git</connection>
<developerConnection>scm:git:git@github.com:apache/incubator-zeppelin.git</developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/incubator-zeppelin.git</url>
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-zeppelin.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-zeppelin.git</developerConnection>
</scm>
<developers>
@ -86,189 +86,40 @@
<modules>
<module>zeppelin-interpreter</module>
<module>zeppelin-zengine</module>
<module>spark-dependencies</module>
<module>spark</module>
<module>markdown</module>
<module>angular</module>
<module>shell</module>
<module>hive</module>
<module>hbase</module>
<module>phoenix</module>
<module>geode</module>
<module>postgresql</module>
<module>tajo</module>
<module>flink</module>
<module>ignite</module>
<module>kylin</module>
<module>lens</module>
<module>cassandra</module>
<module>zeppelin-web</module>
<module>zeppelin-server</module>
<module>zeppelin-distribution</module>
</modules>
<properties>
<spark.version>1.1.1</spark.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<scala.macros.version>2.0.1</scala.macros.version>
<mesos.version>0.18.1</mesos.version>
<mesos.classifier>shaded-protobuf</mesos.classifier>
<akka.group>org.spark-project.akka</akka.group>
<akka.version>2.2.3-shaded-protobuf</akka.version>
<slf4j.version>1.7.10</slf4j.version>
<log4j.version>1.2.17</log4j.version>
<hadoop.version>1.0.4</hadoop.version>
<protobuf.version>2.4.1</protobuf.version>
<yarn.version>${hadoop.version}</yarn.version>
<hbase.version>0.94.6</hbase.version>
<zookeeper.version>3.4.5</zookeeper.version>
<hive.version>0.12.0</hive.version>
<tajo.version>0.10.0</tajo.version>
<derby.version>10.4.2.0</derby.version>
<parquet.version>1.4.3</parquet.version>
<jblas.version>1.2.3</jblas.version>
<jetty.version>8.1.14.v20131031</jetty.version>
<json4s.version>3.2.10</json4s.version>
<chill.version>0.3.6</chill.version>
<codahale.metrics.version>3.0.0</codahale.metrics.version>
<avro.version>1.7.6</avro.version>
<jets3t.version>0.7.1</jets3t.version>
<commons.math3.version>3.2</commons.math3.version>
<commons.httpclient.version>4.3.6</commons.httpclient.version>
<codehaus.jackson.version>1.8.8</codehaus.jackson.version>
<snappy.version>1.0.5</snappy.version>
<io.netty.version>4.0.17.Final</io.netty.version>
<jruby.version>1.6.8</jruby.version>
<libthrift.version>0.9.2</libthrift.version>
<gson.version>2.2</gson.version>
<guava.version>15.0</guava.version>
<scalatest.version>2.2.1</scalatest.version>
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
<java.version>1.7</java.version>
<cxf.version>2.7.7</cxf.version>
<gson.version>2.2</gson.version>
</properties>
<dependencyManagement>
<dependencies>
<!-- Jetty -->
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>jetty-all-server</artifactId>
<version>${jetty.version}</version>
</dependency>
<!-- Commons -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${commons.httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.3.3</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.3.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>${commons.math3.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.5</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.9</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<version>1.3.9</version>
</dependency>
<!-- Logging -->
<dependency>
@ -283,19 +134,6 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jul-to-slf4j</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>${slf4j.version}</version>
<!-- <scope>runtime</scope> --> <!-- more correct, but scalac 2.10.3 doesn't like it -->
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@ -303,243 +141,67 @@
</dependency>
<dependency>
<groupId>com.ning</groupId>
<artifactId>compress-lzf</artifactId>
<version>1.0.0</version>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
<version>${libthrift.version}</version>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.3.3</version>
</dependency>
<dependency>
<groupId>com.clearspring.analytics</groupId>
<artifactId>stream</artifactId>
<version>2.5.1</version>
<exclusions>
<!-- Only HyperLogLog is used, which doesn't depend on fastutil -->
<exclusion>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- In theory we need not directly depend on protobuf since Spark does
not directly use it. However, when building with Hadoop/YARN 2.2 Maven doesn't
correctly bump the protobuf version up from the one Mesos gives. For now
we include this variable to explicitly bump the version when building with
YARN. It would be nice to figure out why Maven can't resolve this correctly
(like SBT does). -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.6</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop-bundle</artifactId>
<version>${parquet.version}</version>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill_${scala.binary.version}</artifactId>
<version>${chill.version}</version>
<exclusions>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-commons</artifactId>
</exclusion>
</exclusions>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>chill-java</artifactId>
<version>${chill.version}</version>
<exclusions>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm-commons</artifactId>
</exclusion>
</exclusions>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.5</version>
</dependency>
<!-- Akka -->
<dependency>
<groupId>${akka.group}</groupId>
<artifactId>akka-actor_${scala.binary.version}</artifactId>
<version>${akka.version}</version>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>${akka.group}</groupId>
<artifactId>akka-remote_${scala.binary.version}</artifactId>
<version>${akka.version}</version>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>${akka.group}</groupId>
<artifactId>akka-slf4j_${scala.binary.version}</artifactId>
<version>${akka.version}</version>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>${akka.group}</groupId>
<artifactId>akka-testkit_${scala.binary.version}</artifactId>
<version>${akka.version}</version>
</dependency>
<dependency>
<groupId>colt</groupId>
<artifactId>colt</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.mesos</groupId>
<artifactId>mesos</artifactId>
<version>${mesos.version}</version>
<classifier>${mesos.classifier}</classifier>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>2.2</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${io.netty.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
<version>3.6.6.Final</version>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<version>${derby.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-jvm</artifactId>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-json</artifactId>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-ganglia</artifactId>
<version>${codahale.metrics.version}</version>
</dependency>
<dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-graphite</artifactId>
<version>${codahale.metrics.version}</version>
</dependency>
<!-- Scala -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>jline</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-actors</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scalap</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>2.1.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymockclassextension</artifactId>
<version>3.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_${scala.binary.version}</artifactId>
<version>1.11.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
@ -547,328 +209,6 @@
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.novocode</groupId>
<artifactId>junit-interface</artifactId>
<version>0.10</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-ipc</artifactId>
<version>${avro.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-mapred</artifactId>
<version>${avro.version}</version>
<exclusions>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
</dependency>
<!-- See SPARK-1556 for info on this dependency: -->
<dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
<version>${jets3t.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<version>${yarn.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
<version>${yarn.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
<version>${yarn.version}</version>
<exclusions>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.ow2.asm</groupId>
<artifactId>asm</artifactId>
</exclusion>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>${codehaus.jackson.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>${codehaus.jackson.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
<version>${codehaus.jackson.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
<version>${codehaus.jackson.version}</version>
</dependency>
<dependency>
<groupId>com.thoughtworks.paranamer</groupId>
<artifactId>paranamer</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>2.2.6</version>
</dependency>
<!-- json4s -->
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-core_2.10</artifactId>
<version>${json4s.version}</version>
</dependency>
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-native_2.10</artifactId>
<version>${json4s.version}</version>
</dependency>
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-jackson_2.10</artifactId>
<version>${json4s.version}</version>
</dependency>
<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-ext_2.10</artifactId>
<version>${json4s.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
@ -898,7 +238,7 @@
<executions>
<execution>
<id>cobertura</id>
<phase>package</phase>
<phase>install</phase>
<goals>
<goal>cobertura</goal>
</goals>
@ -917,10 +257,9 @@
</configuration>
<executions>
<execution>
<id>checkstyle</id>
<id>checkstyle-fail-build</id>
<phase>validate</phase>
<goals>
<goal>checkstyle</goal>
<goal>check</goal>
</goals>
<configuration>
@ -928,6 +267,17 @@
<excludes>org/apache/zeppelin/interpreter/thrift/*</excludes>
</configuration>
</execution>
<execution>
<id>checkstyle-gen-html-report</id>
<phase>install</phase>
<goals>
<goal>checkstyle-aggregate</goal>
</goals>
<configuration>
<excludes>org/apache/zeppelin/interpreter/thrift/*</excludes>
</configuration>
</execution>
</executions>
</plugin>
@ -1055,6 +405,7 @@
<version>0.11</version>
<configuration>
<excludes>
<exclude>**/*.keywords</exclude>
<exclude>**/.idea/</exclude>
<exclude>**/*.iml</exclude>
<exclude>.git/</exclude>
@ -1086,6 +437,7 @@
<exclude>STYLE.md</exclude>
<exclude>Roadmap.md</exclude>
<exclude>conf/interpreter.json</exclude>
<exclude>spark-*-bin*/**</exclude>
</excludes>
</configuration>
@ -1147,7 +499,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-clean-plugin</artifactId>
<version>2.4.1</version>
<version>2.6.1</version>
<configuration>
<filesets>
<fileset>
@ -1289,6 +641,7 @@
</plugins>
</build>
</profile>
<<<<<<< HEAD
<profile>
<id>spark-1.1</id>
@ -1503,5 +856,7 @@
</dependency>
</dependencies>
</profile>
=======
>>>>>>> master
</profiles>
</project>

9
postgresql/README.md Normal file
View file

@ -0,0 +1,9 @@
## PostgreSQL, HAWQ and Greenplum Interpreter for Apache Zeppelin
This interpreter supports the following SQL engines:
* [PostgreSQL](http://www.postgresql.org/) - OSS, Object-relational database management system (ORDBMS)
* [Apache HAWQ](http://pivotal.io/big-data/pivotal-hawq) - Powerful [Open Source](https://wiki.apache.org/incubator/HAWQProposal) SQL-On-Hadoop engine.
* [Greenplum](http://pivotal.io/big-data/pivotal-greenplum-database) - MPP database built on open source PostgreSQL.
The official documentation: [PostgreSQL, HAWQ](https://zeppelin.incubator.apache.org/docs/interpreter/postgresql.html)

162
postgresql/pom.xml Normal file
View file

@ -0,0 +1,162 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-postgresql</artifactId>
<packaging>jar</packaging>
<version>0.6.0-incubating-SNAPSHOT</version>
<name>Zeppelin: PostgreSQL interpreter</name>
<url>http://www.apache.org</url>
<properties>
<postgresql.version>9.4-1201-jdbc41</postgresql.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>${postgresql.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
<version>2.12.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.mockrunner</groupId>
<artifactId>mockrunner-jdbc</artifactId>
<version>1.0.8</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/psql</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/psql</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

337
postgresql/src/main/java/org/apache/zeppelin/postgresql/PostgreSqlInterpreter.java Normal file
View file

@ -0,0 +1,337 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.postgresql;
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
/**
* PostgreSQL interpreter for Zeppelin. This interpreter can also be used for accessing HAWQ and
* GreenplumDB.
*
* <ul>
* <li>{@code postgresql.url} - JDBC URL to connect to.</li>
* <li>{@code postgresql.user} - JDBC user name..</li>
* <li>{@code postgresql.password} - JDBC password..</li>
* <li>{@code postgresql.driver.name} - JDBC driver name.</li>
* <li>{@code postgresql.max.result} - Max number of SQL result to display.</li>
* </ul>
*
* <p>
* How to use: <br/>
* {@code %psql.sql} <br/>
* {@code
* SELECT store_id, count(*)
* FROM retail_demo.order_lineitems_pxf
* GROUP BY store_id;
* }
* </p>
*
* For SQL auto-completion use the (Ctrl+.) shortcut.
*/
public class PostgreSqlInterpreter extends Interpreter {
private Logger logger = LoggerFactory.getLogger(PostgreSqlInterpreter.class);
private static final char WhITESPACE = ' ';
private static final char NEWLINE = '\n';
private static final char TAB = '\t';
private static final String TABLE_MAGIC_TAG = "%table ";
private static final String EXPLAIN_PREDICATE = "EXPLAIN ";
private static final String UPDATE_COUNT_HEADER = "Update Count";
static final String DEFAULT_JDBC_URL = "jdbc:postgresql://localhost:5432/";
static final String DEFAULT_JDBC_USER_PASSWORD = "";
static final String DEFAULT_JDBC_USER_NAME = "gpadmin";
static final String DEFAULT_JDBC_DRIVER_NAME = "org.postgresql.Driver";
static final String DEFAULT_MAX_RESULT = "1000";
static final String POSTGRESQL_SERVER_URL = "postgresql.url";
static final String POSTGRESQL_SERVER_USER = "postgresql.user";
static final String POSTGRESQL_SERVER_PASSWORD = "postgresql.password";
static final String POSTGRESQL_SERVER_DRIVER_NAME = "postgresql.driver.name";
static final String POSTGRESQL_SERVER_MAX_RESULT = "postgresql.max.result";
static {
Interpreter.register(
"sql",
"psql",
PostgreSqlInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(POSTGRESQL_SERVER_URL, DEFAULT_JDBC_URL, "The URL for PostgreSQL.")
.add(POSTGRESQL_SERVER_USER, DEFAULT_JDBC_USER_NAME, "The PostgreSQL user name")
.add(POSTGRESQL_SERVER_PASSWORD, DEFAULT_JDBC_USER_PASSWORD,
"The PostgreSQL user password")
.add(POSTGRESQL_SERVER_DRIVER_NAME, DEFAULT_JDBC_DRIVER_NAME, "JDBC Driver Name")
.add(POSTGRESQL_SERVER_MAX_RESULT, DEFAULT_MAX_RESULT,
"Max number of SQL result to display.").build());
}
private Connection jdbcConnection;
private Statement currentStatement;
private Exception exceptionOnConnect;
private int maxResult;
private SqlCompleter sqlCompleter;
private static final Function<CharSequence, String> sequenceToStringTransformer =
new Function<CharSequence, String>() {
public String apply(CharSequence seq) {
return seq.toString();
}
};
private static final List<String> NO_COMPLETION = new ArrayList<String>();
public PostgreSqlInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
logger.info("Open psql connection!");
// Ensure that no previous connections are left open.
close();
try {
String driverName = getProperty(POSTGRESQL_SERVER_DRIVER_NAME);
String url = getProperty(POSTGRESQL_SERVER_URL);
String user = getProperty(POSTGRESQL_SERVER_USER);
String password = getProperty(POSTGRESQL_SERVER_PASSWORD);
maxResult = Integer.valueOf(getProperty(POSTGRESQL_SERVER_MAX_RESULT));
Class.forName(driverName);
jdbcConnection = DriverManager.getConnection(url, user, password);
sqlCompleter = createSqlCompleter(jdbcConnection);
exceptionOnConnect = null;
logger.info("Successfully created psql connection");
} catch (ClassNotFoundException | SQLException e) {
logger.error("Cannot open connection", e);
exceptionOnConnect = e;
close();
}
}
private SqlCompleter createSqlCompleter(Connection jdbcConnection) {
SqlCompleter completer = null;
try {
Set<String> keywordsCompletions = SqlCompleter.getSqlKeywordsCompletions(jdbcConnection);
Set<String> dataModelCompletions =
SqlCompleter.getDataModelMetadataCompletions(jdbcConnection);
SetView<String> allCompletions = Sets.union(keywordsCompletions, dataModelCompletions);
completer = new SqlCompleter(allCompletions, dataModelCompletions);
} catch (IOException | SQLException e) {
logger.error("Cannot create SQL completer", e);
}
return completer;
}
@Override
public void close() {
logger.info("Close psql connection!");
try {
if (getJdbcConnection() != null) {
getJdbcConnection().close();
}
} catch (SQLException e) {
logger.error("Cannot close connection", e);
} finally {
exceptionOnConnect = null;
}
}
private InterpreterResult executeSql(String sql) {
try {
if (exceptionOnConnect != null) {
return new InterpreterResult(Code.ERROR, exceptionOnConnect.getMessage());
}
currentStatement = getJdbcConnection().createStatement();
currentStatement.setMaxRows(maxResult);
StringBuilder msg = null;
boolean isTableType = false;
if (containsIgnoreCase(sql, EXPLAIN_PREDICATE)) {
msg = new StringBuilder();
} else {
msg = new StringBuilder(TABLE_MAGIC_TAG);
isTableType = true;
}
ResultSet resultSet = null;
try {
boolean isResultSetAvailable = currentStatement.execute(sql);
if (isResultSetAvailable) {
resultSet = currentStatement.getResultSet();
ResultSetMetaData md = resultSet.getMetaData();
for (int i = 1; i < md.getColumnCount() + 1; i++) {
if (i > 1) {
msg.append(TAB);
}
msg.append(replaceReservedChars(isTableType, md.getColumnName(i)));
}
msg.append(NEWLINE);
int displayRowCount = 0;
while (resultSet.next() && displayRowCount < getMaxResult()) {
for (int i = 1; i < md.getColumnCount() + 1; i++) {
msg.append(replaceReservedChars(isTableType, resultSet.getString(i)));
if (i != md.getColumnCount()) {
msg.append(TAB);
}
}
msg.append(NEWLINE);
displayRowCount++;
}
} else {
// Response contains either an update count or there are no results.
int updateCount = currentStatement.getUpdateCount();
msg.append(UPDATE_COUNT_HEADER).append(NEWLINE);
msg.append(updateCount).append(NEWLINE);
// In case of update event (e.g. isResultSetAvailable = false) update the completion
// meta-data.
if (sqlCompleter != null) {
sqlCompleter.updateDataModelMetaData(getJdbcConnection());
}
}
} finally {
try {
if (resultSet != null) {
resultSet.close();
}
currentStatement.close();
} finally {
currentStatement = null;
}
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
} catch (SQLException ex) {
logger.error("Cannot run " + sql, ex);
return new InterpreterResult(Code.ERROR, ex.getMessage());
}
}
/**
* For %table response replace Tab and Newline characters from the content.
*/
private String replaceReservedChars(boolean isTableResponseType, String str) {
return (!isTableResponseType) ? str : str.replace(TAB, WhITESPACE).replace(NEWLINE, WhITESPACE);
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.info("Run SQL command '{}'", cmd);
return executeSql(cmd);
}
@Override
public void cancel(InterpreterContext context) {
logger.info("Cancel current query statement.");
if (currentStatement != null) {
try {
currentStatement.cancel();
} catch (SQLException ex) {
} finally {
currentStatement = null;
}
}
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
PostgreSqlInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
List<CharSequence> candidates = new ArrayList<CharSequence>();
if (sqlCompleter != null && sqlCompleter.complete(buf, cursor, candidates) >= 0) {
return Lists.transform(candidates, sequenceToStringTransformer);
} else {
return NO_COMPLETION;
}
}
public int getMaxResult() {
return maxResult;
}
// Test only method
protected Connection getJdbcConnection() {
return jdbcConnection;
}
}

250
postgresql/src/main/java/org/apache/zeppelin/postgresql/SqlCompleter.java Normal file
View file

@ -0,0 +1,250 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.postgresql;
/*
* This source file is based on code taken from SQLLine 1.0.2 See SQLLine notice in LICENSE
*/
import static org.apache.commons.lang.StringUtils.isBlank;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeSet;
import java.util.regex.Pattern;
import jline.console.completer.ArgumentCompleter.ArgumentList;
import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
import jline.console.completer.StringsCompleter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
/**
* SQL auto complete functionality for the PostgreSqlInterpreter.
*/
public class SqlCompleter extends StringsCompleter {
private static Logger logger = LoggerFactory.getLogger(SqlCompleter.class);
/**
* Delimiter that can split SQL statement in keyword list
*/
private WhitespaceArgumentDelimiter sqlDelimiter = new WhitespaceArgumentDelimiter() {
private Pattern pattern = Pattern.compile("[\\.:;,]");
@Override
public boolean isDelimiterChar(CharSequence buffer, int pos) {
return pattern.matcher("" + buffer.charAt(pos)).matches()
|| super.isDelimiterChar(buffer, pos);
}
};
private Set<String> modelCompletions = new HashSet<String>();
public SqlCompleter(Set<String> allCompletions, Set<String> dataModelCompletions) {
super(allCompletions);
this.modelCompletions = dataModelCompletions;
}
@Override
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
if (isBlank(buffer) || (cursor > buffer.length() + 1)) {
return -1;
}
// The delimiter breaks the buffer into separate words (arguments), separated by the
// white spaces.
ArgumentList argumentList = sqlDelimiter.delimit(buffer, cursor);
String argument = argumentList.getCursorArgument();
// cursor in the selected argument
int argumentPosition = argumentList.getArgumentPosition();
if (isBlank(argument)) {
int argumentsCount = argumentList.getArguments().length;
if (argumentsCount <= 0 || ((buffer.length() + 2) < cursor)
|| sqlDelimiter.isDelimiterChar(buffer, cursor - 2)) {
return -1;
}
argument = argumentList.getArguments()[argumentsCount - 1];
argumentPosition = argument.length();
}
int complete = super.complete(argument, argumentPosition, candidates);
logger.debug("complete:" + complete + ", size:" + candidates.size());
return complete;
}
public void updateDataModelMetaData(Connection connection) {
try {
Set<String> newModelCompletions = getDataModelMetadataCompletions(connection);
logger.debug("New model metadata is:" + Joiner.on(',').join(newModelCompletions));
// Sets.difference(set1, set2) - returned set contains all elements that are contained by set1
// and not contained by set2. set2 may also contain elements not present in set1; these are
// simply ignored.
SetView<String> removedCompletions = Sets.difference(modelCompletions, newModelCompletions);
logger.debug("Removed Model Completions: " + Joiner.on(',').join(removedCompletions));
this.getStrings().removeAll(removedCompletions);
SetView<String> newCompletions = Sets.difference(newModelCompletions, modelCompletions);
logger.debug("New Completions: " + Joiner.on(',').join(newCompletions));
this.getStrings().addAll(newCompletions);
modelCompletions = newModelCompletions;
} catch (SQLException e) {
logger.error("Failed to update the metadata conmpletions", e);
}
}
public static Set<String> getSqlKeywordsCompletions(Connection connection) throws IOException,
SQLException {
// Add the default SQL completions
String keywords =
new BufferedReader(new InputStreamReader(
SqlCompleter.class.getResourceAsStream("/ansi.sql.keywords"))).readLine();
DatabaseMetaData metaData = connection.getMetaData();
// Add the driver specific SQL completions
String driverSpecificKeywords =
"/" + metaData.getDriverName().replace(" ", "-").toLowerCase() + "-sql.keywords";
logger.info("JDBC DriverName:" + driverSpecificKeywords);
if (SqlCompleter.class.getResource(driverSpecificKeywords) != null) {
String driverKeywords =
new BufferedReader(new InputStreamReader(
SqlCompleter.class.getResourceAsStream(driverSpecificKeywords))).readLine();
keywords += "," + driverKeywords.toUpperCase();
}
Set<String> completions = new TreeSet<String>();
// Add the keywords from the current JDBC connection
try {
keywords += "," + metaData.getSQLKeywords();
} catch (Exception e) {
logger.debug("fail to get SQL key words from database metadata: " + e, e);
}
try {
keywords += "," + metaData.getStringFunctions();
} catch (Exception e) {
logger.debug("fail to get string function names from database metadata: " + e, e);
}
try {
keywords += "," + metaData.getNumericFunctions();
} catch (Exception e) {
logger.debug("fail to get numeric function names from database metadata: " + e, e);
}
try {
keywords += "," + metaData.getSystemFunctions();
} catch (Exception e) {
logger.debug("fail to get system function names from database metadata: " + e, e);
}
try {
keywords += "," + metaData.getTimeDateFunctions();
} catch (Exception e) {
logger.debug("fail to get time date function names from database metadata: " + e, e);
}
// Also allow lower-case versions of all the keywords
keywords += "," + keywords.toLowerCase();
StringTokenizer tok = new StringTokenizer(keywords, ", ");
while (tok.hasMoreTokens()) {
completions.add(tok.nextToken());
}
return completions;
}
public static Set<String> getDataModelMetadataCompletions(Connection connection)
throws SQLException {
Set<String> completions = new TreeSet<String>();
getColumnNames(connection.getMetaData(), completions);
getSchemaNames(connection.getMetaData(), completions);
return completions;
}
private static void getColumnNames(DatabaseMetaData meta, Set<String> names) throws SQLException {
try {
ResultSet columns = meta.getColumns(meta.getConnection().getCatalog(), null, "%", "%");
try {
while (columns.next()) {
// Add the following strings: (1) column name, (2) table name
String name = columns.getString("TABLE_NAME");
if (!isBlank(name)) {
names.add(name);
names.add(columns.getString("COLUMN_NAME"));
// names.add(columns.getString("TABLE_NAME") + "." + columns.getString("COLUMN_NAME"));
}
}
} finally {
columns.close();
}
logger.debug(Joiner.on(',').join(names));
} catch (Throwable t) {
logger.error("Failed to retrieve the column name", t);
}
}
private static void getSchemaNames(DatabaseMetaData meta, Set<String> names) throws SQLException {
try {
ResultSet schemas = meta.getSchemas();
try {
while (schemas.next()) {
String schemaName = schemas.getString("TABLE_SCHEM");
if (!isBlank(schemaName)) {
names.add(schemaName + ".");
}
}
} finally {
schemas.close();
}
} catch (Throwable t) {
logger.error("Failed to retrieve the column name", t);
}
}
// test purpose only
WhitespaceArgumentDelimiter getSqlDelimiter() {
return this.sqlDelimiter;
}
}

1
postgresql/src/main/resources/ansi.sql.keywords Normal file
View file

@ -0,0 +1 @@
ABSOLUTE,ACTION,ADD,ALL,ALLOCATE,ALTER,AND,ANY,ARE,AS,ASC,ASSERTION,AT,AUTHORIZATION,AVG,BEGIN,BETWEEN,BIT,BIT_LENGTH,BOTH,BY,CASCADE,CASCADED,CASE,CAST,CATALOG,CHAR,CHARACTER,CHAR_LENGTH,CHARACTER_LENGTH,CHECK,CLOSE,CLUSTER,COALESCE,COLLATE,COLLATION,COLUMN,COMMIT,CONNECT,CONNECTION,CONSTRAINT,CONSTRAINTS,CONTINUE,CONVERT,CORRESPONDING,COUNT,CREATE,CROSS,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATE,DAY,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFERRABLE,DEFERRED,DELETE,DESC,DESCRIBE,DESCRIPTOR,DIAGNOSTICS,DISCONNECT,DISTINCT,DOMAIN,DOUBLE,DROP,ELSE,END,END-EXEC,ESCAPE,EXCEPT,EXCEPTION,EXEC,EXECUTE,EXISTS,EXTERNAL,EXTRACT,FALSE,FETCH,FIRST,FLOAT,FOR,FOREIGN,FOUND,FROM,FULL,GET,GLOBAL,GO,GOTO,GRANT,GROUP,HAVING,HOUR,IDENTITY,IMMEDIATE,IN,INDICATOR,INITIALLY,INNER,INPUT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,IS,ISOLATION,JOIN,KEY,LANGUAGE,LAST,LEADING,LEFT,LEVEL,LIKE,LOCAL,LOWER,MATCH,MAX,MIN,MINUTE,MODULE,MONTH,NAMES,NATIONAL,NATURAL,NCHAR,NEXT,NO,NOT,NULL,NULLIF,NUMERIC,OCTET_LENGTH,OF,ON,ONLY,OPEN,OPTION,OR,ORDER,OUTER,OUTPUT,OVERLAPS,OVERWRITE,PAD,PARTIAL,PARTITION,POSITION,PRECISION,PREPARE,PRESERVE,PRIMARY,PRIOR,PRIVILEGES,PROCEDURE,PUBLIC,READ,REAL,REFERENCES,RELATIVE,RESTRICT,REVOKE,RIGHT,ROLLBACK,ROWS,SCHEMA,SCROLL,SECOND,SECTION,SELECT,SESSION,SESSION_USER,SET,SIZE,SMALLINT,SOME,SPACE,SQL,SQLCODE,SQLERROR,SQLSTATE,SUBSTRING,SUM,SYSTEM_USER,TABLE,TEMPORARY,THEN,TIME,TIMESTAMP,TIMEZONE_HOUR,TIMEZONE_MINUTE,TO,TRAILING,TRANSACTION,TRANSLATE,TRANSLATION,TRIM,TRUE,UNION,UNIQUE,UNKNOWN,UPDATE,UPPER,USAGE,USER,USING,VALUE,VALUES,VARCHAR,VARYING,VIEW,WHEN,WHENEVER,WHERE,WITH,WORK,WRITE,YEAR,ZONE,ADA,C,CATALOG_NAME,CHARACTER_SET_CATALOG,CHARACTER_SET_NAME,CHARACTER_SET_SCHEMA,CLASS_ORIGIN,COBOL,COLLATION_CATALOG,COLLATION_NAME,COLLATION_SCHEMA,COLUMN_NAME,COMMAND_FUNCTION,COMMITTED,CONDITION_NUMBER,CONNECTION_NAME,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CURSOR_NAME,DATA,DATETIME_INTERVAL_CODE,DATETIME_INTERVAL_PRECISION,DYNAMIC_FUNCTION,FORTRAN,LENGTH,MESSAGE_LENGTH,MESSAGE_OCTET_LENGTH,MESSAGE_TEXT,MORE,MUMPS,NAME,NULLABLE,NUMBER,PASCAL,PLI,REPEATABLE,RETURNED_LENGTH,RETURNED_OCTET_LENGTH,RETURNED_SQLSTATE,ROW_COUNT,SCALE,SCHEMA_NAME,SERIALIZABLE,SERVER_NAME,SUBCLASS_ORIGIN,TABLE_NAME,TYPE,UNCOMMITTED,UNNAMED,LIMIT

1
postgresql/src/main/resources/postgresql-native-driver-sql.keywords Normal file
View file

File diff suppressed because one or more lines are too long

239
postgresql/src/test/java/org/apache/zeppelin/postgresql/PostgreSqlInterpreterTest.java Normal file
View file

@ -0,0 +1,239 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.postgresql;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.DEFAULT_JDBC_DRIVER_NAME;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.DEFAULT_JDBC_URL;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.DEFAULT_JDBC_USER_NAME;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.DEFAULT_JDBC_USER_PASSWORD;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.DEFAULT_MAX_RESULT;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.POSTGRESQL_SERVER_DRIVER_NAME;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.POSTGRESQL_SERVER_MAX_RESULT;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.POSTGRESQL_SERVER_PASSWORD;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.POSTGRESQL_SERVER_URL;
import static org.apache.zeppelin.postgresql.PostgreSqlInterpreter.POSTGRESQL_SERVER_USER;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.sql.SQLException;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.Before;
import org.junit.Test;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
import com.mockrunner.jdbc.StatementResultSetHandler;
import com.mockrunner.mock.jdbc.MockConnection;
import com.mockrunner.mock.jdbc.MockResultSet;
/**
* PostgreSQL interpreter unit tests
*/
public class PostgreSqlInterpreterTest extends BasicJDBCTestCaseAdapter {
private PostgreSqlInterpreter psqlInterpreter = null;
private MockResultSet result = null;
@Before
public void beforeTest() {
MockConnection connection = getJDBCMockObjectFactory().getMockConnection();
StatementResultSetHandler statementHandler = connection.getStatementResultSetHandler();
result = statementHandler.createResultSet();
statementHandler.prepareGlobalResultSet(result);
Properties properties = new Properties();
properties.put(POSTGRESQL_SERVER_DRIVER_NAME, DEFAULT_JDBC_DRIVER_NAME);
properties.put(POSTGRESQL_SERVER_URL, DEFAULT_JDBC_URL);
properties.put(POSTGRESQL_SERVER_USER, DEFAULT_JDBC_USER_NAME);
properties.put(POSTGRESQL_SERVER_PASSWORD, DEFAULT_JDBC_USER_PASSWORD);
properties.put(POSTGRESQL_SERVER_MAX_RESULT, DEFAULT_MAX_RESULT);
psqlInterpreter = spy(new PostgreSqlInterpreter(properties));
when(psqlInterpreter.getJdbcConnection()).thenReturn(connection);
}
@Test
public void testOpenCommandIndempotency() throws SQLException {
// Ensure that an attempt to open new connection will clean any remaining connections
psqlInterpreter.open();
psqlInterpreter.open();
psqlInterpreter.open();
verify(psqlInterpreter, times(3)).open();
verify(psqlInterpreter, times(3)).close();
}
@Test
public void testDefaultProperties() throws SQLException {
PostgreSqlInterpreter psqlInterpreter = new PostgreSqlInterpreter(new Properties());
assertEquals(DEFAULT_JDBC_DRIVER_NAME,
psqlInterpreter.getProperty(POSTGRESQL_SERVER_DRIVER_NAME));
assertEquals(DEFAULT_JDBC_URL, psqlInterpreter.getProperty(POSTGRESQL_SERVER_URL));
assertEquals(DEFAULT_JDBC_USER_NAME, psqlInterpreter.getProperty(POSTGRESQL_SERVER_USER));
assertEquals(DEFAULT_JDBC_USER_PASSWORD,
psqlInterpreter.getProperty(POSTGRESQL_SERVER_PASSWORD));
assertEquals(DEFAULT_MAX_RESULT, psqlInterpreter.getProperty(POSTGRESQL_SERVER_MAX_RESULT));
}
@Test
public void testConnectionClose() throws SQLException {
PostgreSqlInterpreter psqlInterpreter = spy(new PostgreSqlInterpreter(new Properties()));
when(psqlInterpreter.getJdbcConnection()).thenReturn(
getJDBCMockObjectFactory().getMockConnection());
psqlInterpreter.close();
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
verifyConnectionClosed();
}
@Test
public void testStatementCancel() throws SQLException {
PostgreSqlInterpreter psqlInterpreter = spy(new PostgreSqlInterpreter(new Properties()));
when(psqlInterpreter.getJdbcConnection()).thenReturn(
getJDBCMockObjectFactory().getMockConnection());
psqlInterpreter.cancel(null);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
assertFalse("Cancel operation should not close the connection", psqlInterpreter
.getJdbcConnection().isClosed());
}
@Test
public void testSelectQuery() throws SQLException {
when(psqlInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
result.addColumn("col2", new String[] {"val21", "val22"});
InterpreterResult interpreterResult = psqlInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("col1\tcol2\nval11\tval21\nval12\tval22\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testSelectQueryMaxResult() throws SQLException {
when(psqlInterpreter.getMaxResult()).thenReturn(1);
String sqlQuery = "select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
result.addColumn("col2", new String[] {"val21", "val22"});
InterpreterResult interpreterResult = psqlInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("col1\tcol2\nval11\tval21\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testSelectQueryWithSpecialCharacters() throws SQLException {
when(psqlInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "select * from t";
result.addColumn("co\tl1", new String[] {"val11", "va\tl1\n2"});
result.addColumn("co\nl2", new String[] {"v\nal21", "val\t22"});
InterpreterResult interpreterResult = psqlInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.type());
assertEquals("co l1\tco l2\nval11\tv al21\nva l1 2\tval 22\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testExplainQuery() throws SQLException {
when(psqlInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "explain select * from t";
result.addColumn("col1", new String[] {"val11", "val12"});
InterpreterResult interpreterResult = psqlInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TEXT, interpreterResult.type());
assertEquals("col1\nval11\nval12\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testExplainQueryWithSpecialCharachters() throws SQLException {
when(psqlInterpreter.getMaxResult()).thenReturn(1000);
String sqlQuery = "explain select * from t";
result.addColumn("co\tl\n1", new String[] {"va\nl11", "va\tl\n12"});
InterpreterResult interpreterResult = psqlInterpreter.interpret(sqlQuery, null);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TEXT, interpreterResult.type());
assertEquals("co\tl\n1\nva\nl11\nva\tl\n12\n", interpreterResult.message());
verifySQLStatementExecuted(sqlQuery);
verifyAllResultSetsClosed();
verifyAllStatementsClosed();
}
@Test
public void testAutoCompletion() throws SQLException {
psqlInterpreter.open();
assertEquals(1, psqlInterpreter.completion("SEL", 0).size());
assertEquals("SELECT ", psqlInterpreter.completion("SEL", 0).iterator().next());
assertEquals(0, psqlInterpreter.completion("SEL", 100).size());
}
}

197
postgresql/src/test/java/org/apache/zeppelin/postgresql/SqlCompleterTest.java Normal file
View file

@ -0,0 +1,197 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.postgresql;
import static com.google.common.collect.Sets.newHashSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Set;
import jline.console.completer.Completer;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
private Logger logger = LoggerFactory.getLogger(SqlCompleterTest.class);
private final static Set<String> EMPTY = new HashSet<String>();
private CompleterTester tester;
private SqlCompleter sqlCompleter;
@Before
public void beforeTest() throws IOException, SQLException {
Set<String> keywordsCompletions =
SqlCompleter.getSqlKeywordsCompletions(getJDBCMockObjectFactory().getMockConnection());
Set<String> dataModelCompletions =
SqlCompleter
.getDataModelMetadataCompletions(getJDBCMockObjectFactory().getMockConnection());
sqlCompleter =
new SqlCompleter(Sets.union(keywordsCompletions, dataModelCompletions),
dataModelCompletions);
tester = new CompleterTester(sqlCompleter);
}
@Test
public void testAfterBufferEnd() {
String buffer = "ORDER";
// Up to 2 white spaces after the buffer end, the completer still uses the last argument
tester.buffer(buffer).from(0).to(buffer.length() + 1).expect(newHashSet("ORDER ")).test();
// 2 white spaces or more behind the buffer end the completer returns empty result
tester.buffer(buffer).from(buffer.length() + 2).to(buffer.length() + 5).expect(EMPTY).test();
}
@Test
public void testEdges() {
String buffer = " ORDER ";
tester.buffer(buffer).from(0).to(8).expect(newHashSet("ORDER ")).test();
tester.buffer(buffer).from(9).to(15).expect(EMPTY).test();
}
@Test
public void testMultipleWords() {
String buffer = " SELE fro LIM";
tester.buffer(buffer).from(0).to(6).expect(newHashSet("SELECT ")).test();
tester.buffer(buffer).from(7).to(11).expect(newHashSet("from ")).test();
tester.buffer(buffer).from(12).to(19).expect(newHashSet("LIMIT ")).test();
tester.buffer(buffer).from(20).to(24).expect(EMPTY).test();
}
@Test
public void testMultiLineBuffer() {
String buffer = " \n SELE \n fro";
tester.buffer(buffer).from(0).to(7).expect(newHashSet("SELECT ")).test();
tester.buffer(buffer).from(8).to(14).expect(newHashSet("from ")).test();
tester.buffer(buffer).from(15).to(17).expect(EMPTY).test();
}
@Test
public void testMultipleCompletionSuggestions() {
String buffer = " SU";
tester.buffer(buffer).from(0).to(5).expect(newHashSet("SUBCLASS_ORIGIN", "SUM", "SUBSTRING"))
.test();
tester.buffer(buffer).from(6).to(7).expect(EMPTY).test();
}
@Test
public void testDotDelimiter() {
String buffer = " order.select ";
tester.buffer(buffer).from(4).to(7).expect(newHashSet("order ")).test();
tester.buffer(buffer).from(8).to(15).expect(newHashSet("select ")).test();
tester.buffer(buffer).from(16).to(17).expect(EMPTY).test();
}
@Test
public void testSqlDelimiterCharacters() {
assertTrue(sqlCompleter.getSqlDelimiter().isDelimiterChar("r.", 1));
assertTrue(sqlCompleter.getSqlDelimiter().isDelimiterChar("SS;", 2));
assertTrue(sqlCompleter.getSqlDelimiter().isDelimiterChar(":", 0));
assertTrue(sqlCompleter.getSqlDelimiter().isDelimiterChar("ttt,", 3));
}
public class CompleterTester {
private Completer completer;
private String buffer;
private int fromCursor;
private int toCursor;
private Set<String> expectedCompletions;
public CompleterTester(Completer completer) {
this.completer = completer;
}
public CompleterTester buffer(String buffer) {
this.buffer = buffer;
return this;
}
public CompleterTester from(int fromCursor) {
this.fromCursor = fromCursor;
return this;
}
public CompleterTester to(int toCursor) {
this.toCursor = toCursor;
return this;
}
public CompleterTester expect(Set<String> expectedCompletions) {
this.expectedCompletions = expectedCompletions;
return this;
}
public void test() {
for (int c = fromCursor; c <= toCursor; c++) {
expectedCompletions(buffer, c, expectedCompletions);
}
}
private void expectedCompletions(String buffer, int cursor, Set<String> expected) {
ArrayList<CharSequence> candidates = new ArrayList<CharSequence>();
completer.complete(buffer, cursor, candidates);
String explain = explain(buffer, cursor, candidates);
logger.info(explain);
assertEquals("Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] "
+ explain, expected, newHashSet(candidates));
}
private String explain(String buffer, int cursor, ArrayList<CharSequence> candidates) {
StringBuffer sb = new StringBuffer();
for (int i = 0; i <= Math.max(cursor, buffer.length()); i++) {
if (i == cursor) {
sb.append("(");
}
if (i >= buffer.length()) {
sb.append("_");
} else {
if (Character.isWhitespace(buffer.charAt(i))) {
sb.append(".");
} else {
sb.append(buffer.charAt(i));
}
}
if (i == cursor) {
sb.append(")");
}
}
sb.append(" >> [").append(Joiner.on(",").join(candidates)).append("]");
return sb.toString();
}
}
}

Some files were not shown because too many files have changed in this diff Show more