Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Adopt fmt-maven-plugin for google-java-format

Browse source 
Adopt all files
This commit is contained in:
Jongyoul Lee 2018-08-21 17:45:04 +09:00
parent f71aca480f
commit f06c50e900
633 changed files with 32256 additions and 28142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

289
_tools/checkstyle.xml
View file

@ -1,289 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
<!-- This is a checkstyle configuration file. For descriptions of what the
following rules do, please see the checkstyle configuration page at http://checkstyle.sourceforge.net/config.html -->
<module name="Checker">
<module name="FileTabCharacter">
<!-- Checks that there are no tab characters in the file. -->
</module>
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf"/>
</module>
<module name="RegexpSingleline">
<!-- Checks that FIXME is not used in comments. TODO is preferred. -->
<property name="format" value="((//.*)|(\*.*))FIXME"/>
<property name="message"
value='TODO is preferred to FIXME. e.g. "TODO(johndoe): Refactor when v2 is released."'/>
</module>
<module name="RegexpSingleline">
<!-- Checks that TODOs are named. (Actually, just that they are followed
by an open paren.) -->
<property name="format" value="((//.*)|(\*.*))TODO[^(]"/>
<property name="message"
value='All TODOs should be named. e.g. "TODO(johndoe): Refactor when v2 is released."'/>
</module>
<!-- <module name="JavadocPackage"> - Checks that each Java package has
a Javadoc file used for commenting. Only allows a package-info.java, not
package.html. </module> -->
<!-- All Java AST specific tests live under TreeWalker module. -->
<module name="TreeWalker">
<!-- IMPORT CHECKS -->
<module name="RedundantImport">
<!-- Checks for redundant import statements. -->
<property name="severity" value="error"/>
</module>
<module name="ImportOrder">
<property name="severity" value="warning"/>
<property name="groups" value="com.google,junit,net,org,java,javax,*,org.apache.zeppelin"/>
<property name="option" value="top"/>
<property name="tokens" value="STATIC_IMPORT, IMPORT"/>
</module>
<!-- JAVADOC CHECKS -->
<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<module name="JavadocMethod">
<property name="scope" value="protected"/>
<property name="severity" value="warning"/>
<property name="allowMissingJavadoc" value="true"/>
<property name="allowMissingParamTags" value="true"/>
<property name="allowMissingReturnTag" value="true"/>
<property name="allowMissingThrowsTags" value="true"/>
<property name="allowThrowsTagsForSubclasses" value="true"/>
<property name="allowUndeclaredRTE" value="true"/>
</module>
<module name="JavadocStyle">
<property name="severity" value="warning"/>
</module>
<!-- NAMING CHECKS -->
<!-- Item 38 - Adhere to generally accepted naming conventions -->
<module name="PackageName">
<!-- Validates identifiers for package names against the supplied expression. -->
<!-- Here the default checkstyle rule restricts package name parts to
seven characters, this is not in line with common practice at Google. -->
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]{1,})*$"/>
<property name="severity" value="warning"/>
</module>
<module name="TypeNameCheck">
<!-- Validates static, final fields against the expression "^[A-Z][a-zA-Z0-9]*$". -->
<metadata name="altname" value="TypeName"/>
<property name="severity" value="warning"/>
</module>
<module name="ConstantNameCheck">
<!-- Validates non-private, static, final fields against the supplied
public/package final fields "^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$". -->
<metadata name="altname" value="ConstantName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="false"/>
<property name="format" value="^([A-Z][A-Z0-9]*(_[A-Z0-9]+)*|FLAG_.*)$"/>
<message key="name.invalidPattern"
value="Variable ''{0}'' should be in ALL_CAPS (if it is a constant) or be private (otherwise)."/>
<property name="severity" value="warning"/>
</module>
<module name="StaticVariableNameCheck">
<!-- Validates static, non-final fields against the supplied expression
"^[a-z][a-zA-Z0-9]*_?$". -->
<metadata name="altname" value="StaticVariableName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="true"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*_?$"/>
<property name="severity" value="warning"/>
</module>
<module name="MemberNameCheck">
<!-- Validates non-static members against the supplied expression. -->
<metadata name="altname" value="MemberName"/>
<property name="applyToPublic" value="true"/>
<property name="applyToProtected" value="true"/>
<property name="applyToPackage" value="true"/>
<property name="applyToPrivate" value="true"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*$"/>
<property name="severity" value="warning"/>
</module>
<module name="MethodNameCheck">
<!-- Validates identifiers for method names. -->
<metadata name="altname" value="MethodName"/>
<property name="format" value="^[a-z][a-zA-Z0-9]*(_[a-zA-Z0-9]+)*$"/>
<property name="severity" value="warning"/>
</module>
<module name="ParameterName">
<!-- Validates identifiers for method parameters against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<module name="LocalFinalVariableName">
<!-- Validates identifiers for local final variables against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<module name="LocalVariableName">
<!-- Validates identifiers for local variables against the expression
"^[a-z][a-zA-Z0-9]*$". -->
<property name="severity" value="warning"/>
</module>
<!-- LENGTH and CODING CHECKS -->
<module name="LineLength">
<!-- Checks if a line is too long. -->
<property name="max"
value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.max}"
default="100"/>
<property name="severity" value="error"/>
<!-- The default ignore pattern exempts the following elements: - import
statements - long URLs inside comments -->
<property name="ignorePattern"
value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.ignorePattern}"
default="^(package .*;\s*)|(import .*;\s*)|( *\* *https?://.*)$"/>
</module>
<module name="LeftCurly">
<!-- Checks for placement of the left curly brace ('{'). -->
<property name="severity" value="warning"/>
</module>
<module name="RightCurly">
<!-- Checks right curlies on CATCH, ELSE, and TRY blocks are on the same
line. e.g., the following example is fine: <pre> if { ... } else </pre> -->
<!-- This next example is not fine: <pre> if { ... } else </pre> -->
<property name="option" value="same"/>
<property name="severity" value="warning"/>
</module>
<!-- Checks for braces around if and else blocks -->
<module name="NeedBraces">
<property name="severity" value="warning"/>
<property name="tokens"
value="LITERAL_IF, LITERAL_ELSE, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO"/>
</module>
<module name="UpperEll">
<!-- Checks that long constants are defined with an upper ell. -->
<property name="severity" value="error"/>
</module>
<module name="FallThrough">
<!-- Warn about falling through to the next case statement. Similar to
javac -Xlint:fallthrough, but the check is suppressed if a single-line comment
on the last non-blank line preceding the fallen-into case contains 'fall
through' (or some other variants which we don't publicized to promote consistency). -->
<property name="reliefPattern"
value="fall through|Fall through|fallthru|Fallthru|falls through|Falls through|fallthrough|Fallthrough|No break|NO break|no break|continue on"/>
<property name="severity" value="error"/>
</module>
<!-- MODIFIERS CHECKS -->
<module name="ModifierOrder">
<!-- Warn if modifier order is inconsistent with JLS3 8.1.1, 8.3.1, and
8.4.3. The prescribed order is: public, protected, private, abstract, static,
final, transient, volatile, synchronized, native, strictfp -->
</module>
<!-- WHITESPACE CHECKS -->
<module name="WhitespaceAround">
<!-- Checks that various tokens are surrounded by whitespace. This includes
most binary operators and keywords followed by regular or curly braces. -->
<property name="tokens"
value="ASSIGN, BAND, BAND_ASSIGN, BOR,
BOR_ASSIGN, BSR, BSR_ASSIGN, BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN,
EQUAL, GE, GT, LAND, LE, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE, LOR, LT, MINUS,
MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL, PLUS, PLUS_ASSIGN, QUESTION,
SL, SL_ASSIGN, SR_ASSIGN, STAR, STAR_ASSIGN"/>
<property name="severity" value="error"/>
</module>
<module name="WhitespaceAfter">
<!-- Checks that commas, semicolons and typecasts are followed by whitespace. -->
<property name="tokens" value="COMMA, SEMI, TYPECAST"/>
</module>
<module name="NoWhitespaceAfter">
<!-- Checks that there is no whitespace after various unary operators.
Linebreaks are allowed. -->
<property name="tokens"
value="BNOT, DEC, DOT, INC, LNOT, UNARY_MINUS,
UNARY_PLUS"/>
<property name="allowLineBreaks" value="true"/>
<property name="severity" value="error"/>
</module>
<module name="NoWhitespaceBefore">
<!-- Checks that there is no whitespace before various unary operators.
Linebreaks are allowed. -->
<property name="tokens" value="SEMI, DOT, POST_DEC, POST_INC"/>
<property name="allowLineBreaks" value="true"/>
<property name="severity" value="error"/>
</module>
<module name="ParenPad">
<!-- Checks that there is no whitespace before close parens or after open
parens. -->
<property name="severity" value="warning"/>
</module>
<module name="Indentation">
<!-- Checks code indentation -->
<property name="basicOffset" value="2"/>
<property name="caseIndent" value="2"/>
</module>
<module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="expected"/>
</module>
<module name="CommentsIndentation"/>
<module name="UnusedImports"/>
<module name="RedundantImport"/>
<module name="RedundantModifier"/>
<module name="AvoidStarImport"/>
</module>
</module>

7
alluxio/pom.xml
View file

@ -137,13 +137,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

217
alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java
View file

@ -1,26 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.alluxio;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import alluxio.Configuration;
import alluxio.shell.AlluxioShell;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
@ -29,22 +24,18 @@ import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import alluxio.Configuration;
import alluxio.shell.AlluxioShell;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Alluxio interpreter for Zeppelin.
*/
/** Alluxio interpreter for Zeppelin. */
public class AlluxioInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(AlluxioInterpreter.class);
protected static final String ALLUXIO_MASTER_HOSTNAME = "alluxio.master.hostname";
@ -58,13 +49,40 @@ public class AlluxioInterpreter extends Interpreter {
private final String alluxioMasterHostname;
private final String alluxioMasterPort;
protected final List<String> keywords = Arrays.asList("cat", "chgrp",
"chmod", "chown", "copyFromLocal", "copyToLocal", "count",
"createLineage", "deleteLineage", "du", "fileInfo", "free",
"getCapacityBytes", "getUsedBytes", "listLineages", "load",
"loadMetadata", "location", "ls", "mkdir", "mount", "mv",
"persist", "pin", "report", "rm", "setTtl", "tail", "touch",
"unmount", "unpin", "unsetTtl");
protected final List<String> keywords =
Arrays.asList(
"cat",
"chgrp",
"chmod",
"chown",
"copyFromLocal",
"copyToLocal",
"count",
"createLineage",
"deleteLineage",
"du",
"fileInfo",
"free",
"getCapacityBytes",
"getUsedBytes",
"listLineages",
"load",
"loadMetadata",
"location",
"ls",
"mkdir",
"mount",
"mv",
"persist",
"pin",
"report",
"rm",
"setTtl",
"tail",
"touch",
"unmount",
"unpin",
"unsetTtl");
public AlluxioInterpreter(Properties property) {
super(property);
@ -75,8 +93,11 @@ public class AlluxioInterpreter extends Interpreter {
@Override
public void open() {
logger.info("Starting Alluxio shell to connect to " + alluxioMasterHostname +
" on port " + alluxioMasterPort);
logger.info(
"Starting Alluxio shell to connect to "
+ alluxioMasterHostname
+ " on port "
+ alluxioMasterPort);
System.setProperty(ALLUXIO_MASTER_HOSTNAME, alluxioMasterHostname);
System.setProperty(ALLUXIO_MASTER_PORT, alluxioMasterPort);
@ -99,18 +120,18 @@ public class AlluxioInterpreter extends Interpreter {
String[] lines = splitAndRemoveEmpty(st, "\n");
return interpret(lines, context);
}
private InterpreterResult interpret(String[] commands, InterpreterContext context) {
boolean isSuccess = true;
totalCommands = commands.length;
completedCommands = 0;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
PrintStream old = System.out;
System.setOut(ps);
for (String command : commands) {
int commandResult = 1;
String[] args = splitAndRemoveEmpty(command, " ");
@ -130,14 +151,14 @@ public class AlluxioInterpreter extends Interpreter {
System.out.flush();
System.setOut(old);
if (isSuccess) {
return new InterpreterResult(Code.SUCCESS, baos.toString());
} else {
return new InterpreterResult(Code.ERROR, baos.toString());
}
}
private String[] splitAndRemoveEmpty(String st, String splitSeparator) {
String[] voices = st.split(splitSeparator);
ArrayList<String> result = new ArrayList<>();
@ -158,7 +179,7 @@ public class AlluxioInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) { }
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
@ -171,15 +192,15 @@ public class AlluxioInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " ");
String lastWord = "";
if (words.length > 0) {
lastWord = words[ words.length - 1 ];
lastWord = words[words.length - 1];
}
List<InterpreterCompletion> voices = new LinkedList<>();
List<InterpreterCompletion> voices = new LinkedList<>();
for (String command : keywords) {
if (command.startsWith(lastWord)) {
voices.add(new InterpreterCompletion(command, command, CompletionType.command.name()));
@ -193,61 +214,81 @@ public class AlluxioInterpreter extends Interpreter {
sb.append("Commands list:");
sb.append("\n\t[help] - List all available commands.");
sb.append("\n\t[cat <path>] - Prints the file's contents to the console.");
sb.append("\n\t[chgrp [-R] <group> <path>] - Changes the group of a file or directory " +
"specified by args. Specify -R to change the group recursively.");
sb.append("\n\t[chmod -R <mode> <path>] - Changes the permission of a file or directory " +
"specified by args. Specify -R to change the permission recursively.");
sb.append("\n\t[chown -R <owner> <path>] - Changes the owner of a file or directory " +
"specified by args. Specify -R to change the owner recursively.");
sb.append("\n\t[copyFromLocal <src> <remoteDst>] - Copies a file or a directory from " +
"local filesystem to Alluxio filesystem.");
sb.append("\n\t[copyToLocal <src> <localDst>] - Copies a file or a directory from the " +
"Alluxio filesystem to the local filesystem.");
sb.append("\n\t[count <path>] - Displays the number of files and directories matching " +
"the specified prefix.");
sb.append("\n\t[createLineage <inputFile1,...> <outputFile1,...> " +
"[<cmd_arg1> <cmd_arg2> ...]] - Creates a lineage.");
sb.append("\n\t[deleteLineage <lineageId> <cascade(true|false)>] - Deletes a lineage. If " +
"cascade is specified as true, dependent lineages will also be deleted.");
sb.append(
"\n\t[chgrp [-R] <group> <path>] - Changes the group of a file or directory "
+ "specified by args. Specify -R to change the group recursively.");
sb.append(
"\n\t[chmod -R <mode> <path>] - Changes the permission of a file or directory "
+ "specified by args. Specify -R to change the permission recursively.");
sb.append(
"\n\t[chown -R <owner> <path>] - Changes the owner of a file or directory "
+ "specified by args. Specify -R to change the owner recursively.");
sb.append(
"\n\t[copyFromLocal <src> <remoteDst>] - Copies a file or a directory from "
+ "local filesystem to Alluxio filesystem.");
sb.append(
"\n\t[copyToLocal <src> <localDst>] - Copies a file or a directory from the "
+ "Alluxio filesystem to the local filesystem.");
sb.append(
"\n\t[count <path>] - Displays the number of files and directories matching "
+ "the specified prefix.");
sb.append(
"\n\t[createLineage <inputFile1,...> <outputFile1,...> "
+ "[<cmd_arg1> <cmd_arg2> ...]] - Creates a lineage.");
sb.append(
"\n\t[deleteLineage <lineageId> <cascade(true|false)>] - Deletes a lineage. If "
+ "cascade is specified as true, dependent lineages will also be deleted.");
sb.append("\n\t[du <path>] - Displays the size of the specified file or directory.");
sb.append("\n\t[fileInfo <path>] - Displays all block info for the specified file.");
sb.append("\n\t[free <file path|folder path>] - Removes the file or directory(recursively) " +
"from Alluxio memory space.");
sb.append(
"\n\t[free <file path|folder path>] - Removes the file or directory(recursively) "
+ "from Alluxio memory space.");
sb.append("\n\t[getCapacityBytes] - Gets the capacity of the Alluxio file system.");
sb.append("\n\t[getUsedBytes] - Gets number of bytes used in the Alluxio file system.");
sb.append("\n\t[listLineages] - Lists all lineages.");
sb.append("\n\t[load <path>] - Loads a file or directory in Alluxio space, makes it " +
"resident in memory.");
sb.append("\n\t[loadMetadata <path>] - Loads metadata for the given Alluxio path from the " +
"under file system.");
sb.append(
"\n\t[load <path>] - Loads a file or directory in Alluxio space, makes it "
+ "resident in memory.");
sb.append(
"\n\t[loadMetadata <path>] - Loads metadata for the given Alluxio path from the "
+ "under file system.");
sb.append("\n\t[location <path>] - Displays the list of hosts storing the specified file.");
sb.append("\n\t[ls [-R] <path>] - Displays information for all files and directories " +
"directly under the specified path. Specify -R to display files and " +
"directories recursively.");
sb.append("\n\t[mkdir <path1> [path2] ... [pathn]] - Creates the specified directories, " +
"including any parent directories that are required.");
sb.append(
"\n\t[ls [-R] <path>] - Displays information for all files and directories "
+ "directly under the specified path. Specify -R to display files and "
+ "directories recursively.");
sb.append(
"\n\t[mkdir <path1> [path2] ... [pathn]] - Creates the specified directories, "
+ "including any parent directories that are required.");
sb.append("\n\t[mount <alluxioPath> <ufsURI>] - Mounts a UFS path onto an Alluxio path.");
sb.append("\n\t[mv <src> <dst>] - Renames a file or directory.");
sb.append("\n\t[persist <alluxioPath>] - Persists a file or directory currently stored " +
"only in Alluxio to the UnderFileSystem.");
sb.append("\n\t[pin <path>] - Pins the given file or directory in memory (works " +
"recursively for directories). Pinned files are never evicted from memory, unless " +
"TTL is set.");
sb.append(
"\n\t[persist <alluxioPath>] - Persists a file or directory currently stored "
+ "only in Alluxio to the UnderFileSystem.");
sb.append(
"\n\t[pin <path>] - Pins the given file or directory in memory (works "
+ "recursively for directories). Pinned files are never evicted from memory, unless "
+ "TTL is set.");
sb.append("\n\t[report <path>] - Reports to the master that a file is lost.");
sb.append("\n\t[rm [-R] <path>] - Removes the specified file. Specify -R to remove file or " +
"directory recursively.");
sb.append("\n\t[setTtl <path> <time to live(in milliseconds)>] - Sets a new TTL value for " +
"the file at path.");
sb.append(
"\n\t[rm [-R] <path>] - Removes the specified file. Specify -R to remove file or "
+ "directory recursively.");
sb.append(
"\n\t[setTtl <path> <time to live(in milliseconds)>] - Sets a new TTL value for "
+ "the file at path.");
sb.append("\n\t[tail <path>] - Prints the file's last 1KB of contents to the console.");
sb.append("\n\t[touch <path>] - Creates a 0 byte file. The file will be written to the " +
"under file system.");
sb.append(
"\n\t[touch <path>] - Creates a 0 byte file. The file will be written to the "
+ "under file system.");
sb.append("\n\t[unmount <alluxioPath>] - Unmounts an Alluxio path.");
sb.append("\n\t[unpin <path>] - Unpins the given file or folder from memory " +
"(works recursively for a directory).");
sb.append(
"\n\t[unpin <path>] - Unpins the given file or folder from memory "
+ "(works recursively for a directory).");
sb.append("\n\\t[unsetTtl <path>] - Unsets the TTL value for the given path.");
sb.append("\n\t[unpin <path>] - Unpin the given file to allow Alluxio to evict " +
"this file again. If the given path is a directory, it recursively unpins " +
"all files contained and any new files created within this directory.");
sb.append(
"\n\t[unpin <path>] - Unpin the given file to allow Alluxio to evict "
+ "this file again. If the given path is a directory, it recursively unpins "
+ "all files contained and any new files created within this directory.");
return sb.toString();
}
}

302
alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java
View file

@ -1,37 +1,19 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.alluxio;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import alluxio.AlluxioURI;
import alluxio.Constants;
import alluxio.client.FileSystemTestUtils;
@ -46,11 +28,22 @@ import alluxio.shell.command.CommandUtils;
import alluxio.util.FormatUtils;
import alluxio.util.io.BufferUtils;
import alluxio.util.io.PathUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class AlluxioInterpreterTest {
private AlluxioInterpreter alluxioInterpreter;
@ -81,26 +74,29 @@ public class AlluxioInterpreterTest {
@Test
public void testCompletion() {
List expectedResultOne = Arrays.asList(
new InterpreterCompletion("cat", "cat", CompletionType.command.name()),
new InterpreterCompletion("chgrp", "chgrp", CompletionType.command.name()),
new InterpreterCompletion("chmod", "chmod", CompletionType.command.name()),
new InterpreterCompletion("chown", "chown", CompletionType.command.name()),
new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
new InterpreterCompletion("count", "count", CompletionType.command.name()),
new InterpreterCompletion("createLineage", "createLineage", CompletionType.command.name()));
List expectedResultTwo = Arrays.asList(
new InterpreterCompletion("copyFromLocal", "copyFromLocal",
CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal",
CompletionType.command.name()),
new InterpreterCompletion("count", "count", CompletionType.command.name()));
List expectedResultThree = Arrays.asList(
new InterpreterCompletion("copyFromLocal", "copyFromLocal",
CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal",
CompletionType.command.name()));
List expectedResultOne =
Arrays.asList(
new InterpreterCompletion("cat", "cat", CompletionType.command.name()),
new InterpreterCompletion("chgrp", "chgrp", CompletionType.command.name()),
new InterpreterCompletion("chmod", "chmod", CompletionType.command.name()),
new InterpreterCompletion("chown", "chown", CompletionType.command.name()),
new InterpreterCompletion(
"copyFromLocal", "copyFromLocal", CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
new InterpreterCompletion("count", "count", CompletionType.command.name()),
new InterpreterCompletion(
"createLineage", "createLineage", CompletionType.command.name()));
List expectedResultTwo =
Arrays.asList(
new InterpreterCompletion(
"copyFromLocal", "copyFromLocal", CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
new InterpreterCompletion("count", "count", CompletionType.command.name()));
List expectedResultThree =
Arrays.asList(
new InterpreterCompletion(
"copyFromLocal", "copyFromLocal", CompletionType.command.name()),
new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()));
List expectedResultNone = new ArrayList<>();
List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0, null);
@ -123,11 +119,11 @@ public class AlluxioInterpreterTest {
@Test
public void catDirectoryTest() throws IOException {
String expected = "Successfully created directory /testDir\n\n" +
"Path /testDir must be a file\n";
String expected =
"Successfully created directory /testDir\n\n" + "Path /testDir must be a file\n";
InterpreterResult output = alluxioInterpreter.interpret("mkdir /testDir" +
"\ncat /testDir", null);
InterpreterResult output =
alluxioInterpreter.interpret("mkdir /testDir" + "\ncat /testDir", null);
Assert.assertEquals(Code.ERROR, output.code());
Assert.assertEquals(expected, output.message().get(0).getData());
@ -141,16 +137,20 @@ public class AlluxioInterpreterTest {
@Test
public void catTest() throws IOException {
FileSystemTestUtils.createByteFile(fs, "/testFile", WriteType.MUST_CACHE,
10, 10);
FileSystemTestUtils.createByteFile(fs, "/testFile", WriteType.MUST_CACHE, 10, 10);
InterpreterResult output = alluxioInterpreter.interpret("cat /testFile", null);
byte[] expected = BufferUtils.getIncreasingByteArray(10);
Assert.assertEquals(Code.SUCCESS, output.code());
Assert.assertArrayEquals(expected,
output.message().get(0).getData().substring(0,
output.message().get(0).getData().length() - 1).getBytes());
Assert.assertArrayEquals(
expected,
output
.message()
.get(0)
.getData()
.substring(0, output.message().get(0).getData().length() - 1)
.getBytes());
}
@Test
@ -162,11 +162,12 @@ public class AlluxioInterpreterTest {
fos.write(toWrite);
fos.close();
InterpreterResult output = alluxioInterpreter.interpret("copyFromLocal " +
testFile.getAbsolutePath() + " /testFile", null);
InterpreterResult output =
alluxioInterpreter.interpret(
"copyFromLocal " + testFile.getAbsolutePath() + " /testFile", null);
Assert.assertEquals(
"Copied " + testFile.getAbsolutePath() + " to /testFile\n\n",
output.message().get(0).getData());
"Copied " + testFile.getAbsolutePath() + " to /testFile\n\n",
output.message().get(0).getData());
long fileLength = fs.getStatus(new AlluxioURI("/testFile")).getLength();
Assert.assertEquals(SIZE_BYTES, fileLength);
@ -195,10 +196,10 @@ public class AlluxioInterpreterTest {
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WriteType.CACHE_THROUGH, 10, 10);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileB", WriteType.MUST_CACHE, 10, 10);
int memPercentageA = fs.getStatus(
new AlluxioURI("/testRoot/testFileA")).getInMemoryPercentage();
int memPercentageB = fs.getStatus(
new AlluxioURI("/testRoot/testFileB")).getInMemoryPercentage();
int memPercentageA =
fs.getStatus(new AlluxioURI("/testRoot/testFileA")).getInMemoryPercentage();
int memPercentageB =
fs.getStatus(new AlluxioURI("/testRoot/testFileB")).getInMemoryPercentage();
Assert.assertFalse(memPercentageA == 0);
Assert.assertTrue(memPercentageB == 100);
@ -217,16 +218,15 @@ public class AlluxioInterpreterTest {
File testDirInner = new File(mLocalAlluxioCluster.getAlluxioHome() + "/testDir/testDirInner");
testDirInner.mkdir();
File testFile =
generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
generateFileContent("/testDir/testDirInner/testFile2",
BufferUtils.getIncreasingByteArray(10, 20));
generateFileContent(
"/testDir/testDirInner/testFile2", BufferUtils.getIncreasingByteArray(10, 20));
InterpreterResult output = alluxioInterpreter.interpret("copyFromLocal " +
testFile.getParent() + " /testDir", null);
InterpreterResult output =
alluxioInterpreter.interpret("copyFromLocal " + testFile.getParent() + " /testDir", null);
Assert.assertEquals(
"Copied " + testFile.getParent() + " to /testDir\n\n",
output.message().get(0).getData());
"Copied " + testFile.getParent() + " to /testDir\n\n", output.message().get(0).getData());
long fileLength1 = fs.getStatus(new AlluxioURI("/testDir/testFile")).getLength();
long fileLength2 = fs.getStatus(new AlluxioURI("/testDir/testDirInner/testFile2")).getLength();
@ -246,14 +246,17 @@ public class AlluxioInterpreterTest {
@Test
public void copyFromLocalTestWithFullURI() throws IOException, AlluxioException {
File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
String uri = "tachyon://" + mLocalAlluxioCluster.getMasterHostname() + ":"
+ mLocalAlluxioCluster.getMasterPort() + "/destFileURI";
String uri =
"tachyon://"
+ mLocalAlluxioCluster.getMasterHostname()
+ ":"
+ mLocalAlluxioCluster.getMasterPort()
+ "/destFileURI";
InterpreterResult output = alluxioInterpreter.interpret("copyFromLocal " +
testFile.getPath() + " " + uri, null);
InterpreterResult output =
alluxioInterpreter.interpret("copyFromLocal " + testFile.getPath() + " " + uri, null);
Assert.assertEquals(
"Copied " + testFile.getPath() + " to " + uri + "\n\n",
output.message().get(0).getData());
"Copied " + testFile.getPath() + " to " + uri + "\n\n", output.message().get(0).getData());
long fileLength = fs.getStatus(new AlluxioURI("/destFileURI")).getLength();
Assert.assertEquals(10L, fileLength);
@ -296,12 +299,13 @@ public class AlluxioInterpreterTest {
private void copyToLocalWithBytes(int bytes) throws IOException {
FileSystemTestUtils.createByteFile(fs, "/testFile", WriteType.MUST_CACHE, 10, 10);
InterpreterResult output = alluxioInterpreter.interpret("copyToLocal /testFile " +
mLocalAlluxioCluster.getAlluxioHome() + "/testFile", null);
InterpreterResult output =
alluxioInterpreter.interpret(
"copyToLocal /testFile " + mLocalAlluxioCluster.getAlluxioHome() + "/testFile", null);
Assert.assertEquals(
"Copied /testFile to " + mLocalAlluxioCluster.getAlluxioHome() + "/testFile\n\n",
output.message().get(0).getData());
"Copied /testFile to " + mLocalAlluxioCluster.getAlluxioHome() + "/testFile\n\n",
output.message().get(0).getData());
fileReadTest("/testFile", 10);
}
@ -309,18 +313,17 @@ public class AlluxioInterpreterTest {
public void countNotExistTest() throws IOException {
InterpreterResult output = alluxioInterpreter.interpret("count /NotExistFile", null);
Assert.assertEquals(Code.ERROR, output.code());
Assert.assertEquals(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message().get(0).getData());
Assert.assertEquals(
ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message().get(0).getData());
}
@Test
public void countTest() throws IOException {
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA",
WriteType.CACHE_THROUGH, 10, 10);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB",
WriteType.CACHE_THROUGH, 20, 20);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileB",
WriteType.CACHE_THROUGH, 30, 30);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WriteType.CACHE_THROUGH, 10, 10);
FileSystemTestUtils.createByteFile(
fs, "/testRoot/testDir/testFileB", WriteType.CACHE_THROUGH, 20, 20);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileB", WriteType.CACHE_THROUGH, 30, 30);
InterpreterResult output = alluxioInterpreter.interpret("count /testRoot", null);
@ -335,16 +338,18 @@ public class AlluxioInterpreterTest {
@Test
public void fileinfoNotExistTest() throws IOException {
InterpreterResult output = alluxioInterpreter.interpret("fileInfo /NotExistFile", null);
Assert.assertEquals(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message().get(0).getData());
Assert.assertEquals(
ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message().get(0).getData());
Assert.assertEquals(Code.ERROR, output.code());
}
@Test
public void locationNotExistTest() throws IOException {
InterpreterResult output = alluxioInterpreter.interpret("location /NotExistFile", null);
Assert.assertEquals(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message().get(0).getData());
Assert.assertEquals(
ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/NotExistFile") + "\n",
output.message().get(0).getData());
Assert.assertEquals(Code.ERROR, output.code());
}
@ -352,12 +357,10 @@ public class AlluxioInterpreterTest {
public void lsTest() throws IOException, AlluxioException {
URIStatus[] files = new URIStatus[3];
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA",
WriteType.MUST_CACHE, 10, 10);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB",
WriteType.MUST_CACHE, 20, 20);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC",
WriteType.THROUGH, 30, 30);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WriteType.MUST_CACHE, 10, 10);
FileSystemTestUtils.createByteFile(
fs, "/testRoot/testDir/testFileB", WriteType.MUST_CACHE, 20, 20);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WriteType.THROUGH, 30, 30);
files[0] = fs.getStatus(new AlluxioURI("/testRoot/testFileA"));
files[1] = fs.getStatus(new AlluxioURI("/testRoot/testDir"));
@ -367,13 +370,26 @@ public class AlluxioInterpreterTest {
String expected = "";
String format = "%-10s%-25s%-15s%-5s\n";
expected += String.format(format, FormatUtils.getSizeFromBytes(10),
CommandUtils.convertMsToDate(files[0].getCreationTimeMs()), "In Memory",
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(10),
CommandUtils.convertMsToDate(files[0].getCreationTimeMs()),
"In Memory",
"/testRoot/testFileA");
expected += String.format(format, FormatUtils.getSizeFromBytes(0),
CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "", "/testRoot/testDir");
expected += String.format(format, FormatUtils.getSizeFromBytes(30),
CommandUtils.convertMsToDate(files[2].getCreationTimeMs()), "Not In Memory",
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(0),
CommandUtils.convertMsToDate(files[1].getCreationTimeMs()),
"",
"/testRoot/testDir");
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(30),
CommandUtils.convertMsToDate(files[2].getCreationTimeMs()),
"Not In Memory",
"/testRoot/testFileC");
expected += "\n";
@ -385,12 +401,10 @@ public class AlluxioInterpreterTest {
public void lsRecursiveTest() throws IOException, AlluxioException {
URIStatus[] files = new URIStatus[4];
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA",
WriteType.MUST_CACHE, 10, 10);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB",
WriteType.MUST_CACHE, 20, 20);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC",
WriteType.THROUGH, 30, 30);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WriteType.MUST_CACHE, 10, 10);
FileSystemTestUtils.createByteFile(
fs, "/testRoot/testDir/testFileB", WriteType.MUST_CACHE, 20, 20);
FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WriteType.THROUGH, 30, 30);
files[0] = fs.getStatus(new AlluxioURI("/testRoot/testFileA"));
files[1] = fs.getStatus(new AlluxioURI("/testRoot/testDir"));
@ -402,21 +416,33 @@ public class AlluxioInterpreterTest {
String expected = "";
String format = "%-10s%-25s%-15s%-5s\n";
expected +=
String.format(format, FormatUtils.getSizeFromBytes(10),
CommandUtils.convertMsToDate(files[0].getCreationTimeMs()), "In Memory",
"/testRoot/testFileA");
String.format(
format,
FormatUtils.getSizeFromBytes(10),
CommandUtils.convertMsToDate(files[0].getCreationTimeMs()),
"In Memory",
"/testRoot/testFileA");
expected +=
String.format(format, FormatUtils.getSizeFromBytes(0),
CommandUtils.convertMsToDate(files[1].getCreationTimeMs()), "",
"/testRoot/testDir");
String.format(
format,
FormatUtils.getSizeFromBytes(0),
CommandUtils.convertMsToDate(files[1].getCreationTimeMs()),
"",
"/testRoot/testDir");
expected +=
String.format(format, FormatUtils.getSizeFromBytes(20),
CommandUtils.convertMsToDate(files[2].getCreationTimeMs()), "In Memory",
"/testRoot/testDir/testFileB");
String.format(
format,
FormatUtils.getSizeFromBytes(20),
CommandUtils.convertMsToDate(files[2].getCreationTimeMs()),
"In Memory",
"/testRoot/testDir/testFileB");
expected +=
String.format(format, FormatUtils.getSizeFromBytes(30),
CommandUtils.convertMsToDate(files[3].getCreationTimeMs()), "Not In Memory",
"/testRoot/testFileC");
String.format(
format,
FormatUtils.getSizeFromBytes(30),
CommandUtils.convertMsToDate(files[3].getCreationTimeMs()),
"Not In Memory",
"/testRoot/testFileC");
expected += "\n";
Assert.assertEquals(expected, output.message().get(0).getData());
@ -424,13 +450,13 @@ public class AlluxioInterpreterTest {
@Test
public void mkdirComplexPathTest() throws IOException, AlluxioException {
InterpreterResult output = alluxioInterpreter.interpret(
"mkdir /Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File", null);
InterpreterResult output =
alluxioInterpreter.interpret("mkdir /Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File", null);
boolean existsDir = fs.exists(new AlluxioURI("/Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File"));
Assert.assertEquals(
"Successfully created directory /Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File\n\n",
output.message().get(0).getData());
"Successfully created directory /Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File\n\n",
output.message().get(0).getData());
Assert.assertTrue(existsDir);
}
@ -444,8 +470,7 @@ public class AlluxioInterpreterTest {
@Test
public void mkdirInvalidPathTest() throws IOException {
Assert.assertEquals(
Code.ERROR,
alluxioInterpreter.interpret("mkdir /test File Invalid Path", null).code());
Code.ERROR, alluxioInterpreter.interpret("mkdir /test File Invalid Path", null).code());
}
@Test
@ -453,26 +478,27 @@ public class AlluxioInterpreterTest {
InterpreterResult output = alluxioInterpreter.interpret("mkdir /root/testFile1", null);
boolean existsDir = fs.exists(new AlluxioURI("/root/testFile1"));
Assert.assertEquals(
"Successfully created directory /root/testFile1\n\n",
output.message().get(0).getData());
"Successfully created directory /root/testFile1\n\n", output.message().get(0).getData());
Assert.assertTrue(existsDir);
}
@Test
public void mkdirTest() throws IOException, AlluxioException {
String qualifiedPath =
"tachyon://" + mLocalAlluxioCluster.getMasterHostname() + ":"
+ mLocalAlluxioCluster.getMasterPort() + "/root/testFile1";
"tachyon://"
+ mLocalAlluxioCluster.getMasterHostname()
+ ":"
+ mLocalAlluxioCluster.getMasterPort()
+ "/root/testFile1";
InterpreterResult output = alluxioInterpreter.interpret("mkdir " + qualifiedPath, null);
boolean existsDir = fs.exists(new AlluxioURI("/root/testFile1"));
Assert.assertEquals(
"Successfully created directory " + qualifiedPath + "\n\n",
output.message().get(0).getData());
"Successfully created directory " + qualifiedPath + "\n\n",
output.message().get(0).getData());
Assert.assertTrue(existsDir);
}
private File generateFileContent(String path, byte[] toWrite)
throws IOException {
private File generateFileContent(String path, byte[] toWrite) throws IOException {
File testFile = new File(mLocalAlluxioCluster.getAlluxioHome() + path);
testFile.createNewFile();
FileOutputStream fos = new FileOutputStream(testFile);

7
angular/pom.xml
View file

@ -72,13 +72,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

22
angular/src/main/java/org/apache/zeppelin/angular/AngularInterpreter.java
View file

@ -20,7 +20,6 @@ package org.apache.zeppelin.angular;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -30,9 +29,7 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
/**
*
*/
/** */
public class AngularInterpreter extends Interpreter {
public AngularInterpreter(Properties property) {
@ -40,12 +37,10 @@ public class AngularInterpreter extends Interpreter {
}
@Override
public void open() {
}
public void open() {}
@Override
public void close() {
}
public void close() {}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
@ -53,8 +48,7 @@ public class AngularInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {
}
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
@ -67,14 +61,14 @@ public class AngularInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return new LinkedList<>();
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
AngularInterpreter.class.getName() + this.hashCode());
return SchedulerFactory.singleton()
.createOrGetFIFOScheduler(AngularInterpreter.class.getName() + this.hashCode());
}
}

7
beam/pom.xml
View file

@ -259,13 +259,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

8
beam/src/main/java/org/apache/zeppelin/beam/BeamInterpreter.java
View file

@ -17,17 +17,13 @@
package org.apache.zeppelin.beam;
import java.util.Properties;
import org.apache.zeppelin.java.JavaInterpreter;
import java.util.Properties;
/**
* Beam interpreter
*/
/** Beam interpreter */
public class BeamInterpreter extends JavaInterpreter {
public BeamInterpreter(Properties property) {
super(property);
}
}

7
bigquery/pom.xml
View file

@ -133,13 +133,6 @@
</descriptorRefs>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

106
bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java
View file

@ -36,10 +36,6 @@ import com.google.api.services.bigquery.model.TableCell;
import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableRow;
import com.google.common.base.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@ -47,7 +43,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -55,35 +50,37 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* BigQuery interpreter for Zeppelin.
*
*
* <ul>
* <li>{@code zeppelin.bigquery.project_id} - Project ID in GCP</li>
* <li>{@code zeppelin.bigquery.wait_time} - Query Timeout in ms</li>
* <li>{@code zeppelin.bigquery.max_no_of_rows} - Max Result size</li>
* </ul>
*
*
* <p>
* How to use: <br/>
* {@code %bigquery.sql<br/>
* {@code
* SELECT departure_airport,count(case when departure_delay>0 then 1 else 0 end) as no_of_delays
* FROM [bigquery-samples:airline_ontime_data.flights]
* group by departure_airport
* order by 2 desc
* SELECT departure_airport,count(case when departure_delay>0 then 1 else 0 end) as no_of_delays
* FROM [bigquery-samples:airline_ontime_data.flights]
* group by departure_airport
* order by 2 desc
* limit 10
* }
* </p>
*
*
*/
public class BigQueryInterpreter extends Interpreter {
private static Logger logger = LoggerFactory.getLogger(BigQueryInterpreter.class);
private static final char NEWLINE = '\n';
private static final char TAB = '\t';
private static Bigquery service = null;
//Mutex created to create the singleton in thread-safe fashion.
// Mutex created to create the singleton in thread-safe fashion.
private static Object serviceLock = new Object();
static final String PROJECT_ID = "zeppelin.bigquery.project_id";
@ -99,16 +96,16 @@ public class BigQueryInterpreter extends Interpreter {
private static final Function<CharSequence, String> sequenceToStringTransformer =
new Function<CharSequence, String>() {
public String apply(CharSequence seq) {
return seq.toString();
}
};
public String apply(CharSequence seq) {
return seq.toString();
}
};
public BigQueryInterpreter(Properties property) {
super(property);
}
//Function to return valid BigQuery Service
// Function to return valid BigQuery Service
@Override
public void open() {
if (service == null) {
@ -120,7 +117,7 @@ public class BigQueryInterpreter extends Interpreter {
logger.info("Opened BigQuery SQL Connection");
} catch (IOException e) {
logger.error("Cannot open connection", e);
exceptionOnConnect = e;
exceptionOnConnect = e;
close();
}
}
@ -128,11 +125,11 @@ public class BigQueryInterpreter extends Interpreter {
}
}
//Function that Creates an authorized client to Google Bigquery.
// Function that Creates an authorized client to Google Bigquery.
private static Bigquery createAuthorizedClient() throws IOException {
HttpTransport transport = new NetHttpTransport();
JsonFactory jsonFactory = new JacksonFactory();
GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);
GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);
if (credential.createScopedRequired()) {
Collection<String> bigqueryScopes = BigqueryScopes.all();
@ -140,15 +137,16 @@ public class BigQueryInterpreter extends Interpreter {
}
return new Bigquery.Builder(transport, jsonFactory, credential)
.setApplicationName("Zeppelin/1.0 (GPN:Apache Zeppelin;)").build();
.setApplicationName("Zeppelin/1.0 (GPN:Apache Zeppelin;)")
.build();
}
//Function that generates and returns the schema and the rows as string
// Function that generates and returns the schema and the rows as string
public static String printRows(final GetQueryResultsResponse response) {
StringBuilder msg = new StringBuilder();
try {
List<String> schemNames = new ArrayList<String>();
for (TableFieldSchema schem: response.getSchema().getFields()) {
for (TableFieldSchema schem : response.getSchema().getFields()) {
schemNames.add(schem.getName());
}
msg.append(Joiner.on(TAB).join(schemNames));
@ -167,32 +165,34 @@ public class BigQueryInterpreter extends Interpreter {
}
}
//Function to poll a job for completion. Future use
public static Job pollJob(final Bigquery.Jobs.Get request, final long interval)
// Function to poll a job for completion. Future use
public static Job pollJob(final Bigquery.Jobs.Get request, final long interval)
throws IOException, InterruptedException {
Job job = request.execute();
while (!job.getStatus().getState().equals("DONE")) {
System.out.println("Job is "
+ job.getStatus().getState()
+ " waiting " + interval + " milliseconds...");
System.out.println(
"Job is " + job.getStatus().getState() + " waiting " + interval + " milliseconds...");
Thread.sleep(interval);
job = request.execute();
}
return job;
}
//Function to page through the results of an arbitrary bigQuery request
// Function to page through the results of an arbitrary bigQuery request
public static <T extends GenericJson> Iterator<T> getPages(
final BigqueryRequest<T> requestTemplate) {
class PageIterator implements Iterator<T> {
private BigqueryRequest<T> request;
private boolean hasNext = true;
PageIterator(final BigqueryRequest<T> requestTemplate) {
this.request = requestTemplate;
}
public boolean hasNext() {
return hasNext;
}
public T next() {
if (!hasNext) {
throw new NoSuchElementException();
@ -217,8 +217,8 @@ public class BigQueryInterpreter extends Interpreter {
return new PageIterator(requestTemplate);
}
//Function to call bigQuery to run SQL and return results to the Interpreter for output
// Function to call bigQuery to run SQL and return results to the Interpreter for output
private InterpreterResult executeSql(String sql) {
int counter = 0;
StringBuilder finalmessage = null;
@ -256,25 +256,31 @@ public class BigQueryInterpreter extends Interpreter {
}
}
//Function to run the SQL on bigQuery service
public static Iterator<GetQueryResultsResponse> run(final String queryString,
final String projId, final long wTime, final long maxRows, Boolean useLegacySql)
throws IOException {
// Function to run the SQL on bigQuery service
public static Iterator<GetQueryResultsResponse> run(
final String queryString,
final String projId,
final long wTime,
final long maxRows,
Boolean useLegacySql)
throws IOException {
try {
logger.info("Use legacy sql: {}", useLegacySql);
QueryResponse query;
query = service
.jobs()
.query(
projId,
new QueryRequest().setTimeoutMs(wTime)
.setUseLegacySql(useLegacySql).setQuery(queryString)
.setMaxResults(maxRows)).execute();
query =
service
.jobs()
.query(
projId,
new QueryRequest()
.setTimeoutMs(wTime)
.setUseLegacySql(useLegacySql)
.setQuery(queryString)
.setMaxResults(maxRows))
.execute();
jobId = query.getJobReference().getJobId();
projectId = query.getJobReference().getProjectId();
GetQueryResults getRequest = service.jobs().getQueryResults(
projectId,
jobId);
GetQueryResults getRequest = service.jobs().getQueryResults(projectId, jobId);
return getPages(getRequest);
} catch (IOException ex) {
throw ex;
@ -296,8 +302,8 @@ public class BigQueryInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
BigQueryInterpreter.class.getName() + this.hashCode());
return SchedulerFactory.singleton()
.createOrGetFIFOScheduler(BigQueryInterpreter.class.getName() + this.hashCode());
}
@Override
@ -329,8 +335,8 @@ public class BigQueryInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return NO_COMPLETION;
}
}

13
bigquery/src/test/java/org/apache/zeppelin/bigquery/BigQueryInterpreterTest.java
View file

@ -21,18 +21,15 @@ import static org.junit.Assert.assertEquals;
import com.google.gson.Gson;
import com.google.gson.JsonIOException;
import com.google.gson.JsonSyntaxException;
import org.junit.Before;
import org.junit.Test;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.Before;
import org.junit.Test;
public class BigQueryInterpreterTest {
protected static class Constants {
@ -48,7 +45,7 @@ public class BigQueryInterpreterTest {
return oneQuery;
}
public String getWrong() {
public String getWrong() {
return wrongQuery;
}
}
@ -98,8 +95,8 @@ public class BigQueryInterpreterTest {
@Test
public void testWithQueryPrefix() {
InterpreterResult ret = bqInterpreter.interpret(
"#standardSQL\n WITH t AS (select 1) SELECT * FROM t", context);
InterpreterResult ret =
bqInterpreter.interpret("#standardSQL\n WITH t AS (select 1) SELECT * FROM t", context);
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
}

7
cassandra/pom.xml
View file

@ -249,13 +249,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

128
cassandra/src/main/java/org/apache/zeppelin/cassandra/CassandraInterpreter.java
View file

@ -18,9 +18,10 @@ package org.apache.zeppelin.cassandra;
import static java.lang.Integer.parseInt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.JdkSSLOptions;
import com.datastax.driver.core.ProtocolOptions.Compression;
import com.datastax.driver.core.Session;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
@ -28,31 +29,24 @@ import java.security.KeyStore;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.JdkSSLOptions;
import com.datastax.driver.core.ProtocolOptions.Compression;
import com.datastax.driver.core.Session;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Interpreter for Apache Cassandra CQL query language.
*/
/** Interpreter for Apache Cassandra CQL query language. */
public class CassandraInterpreter extends Interpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(CassandraInterpreter.class);
public static final String CASSANDRA_INTERPRETER_PARALLELISM =
"cassandra.interpreter.parallelism";
"cassandra.interpreter.parallelism";
public static final String CASSANDRA_HOSTS = "cassandra.hosts";
public static final String CASSANDRA_PORT = "cassandra.native.port";
public static final String CASSANDRA_PROTOCOL_VERSION = "cassandra.protocol.version";
@ -65,62 +59,54 @@ public class CassandraInterpreter extends Interpreter {
public static final String CASSANDRA_RETRY_POLICY = "cassandra.retry.policy";
public static final String CASSANDRA_RECONNECTION_POLICY = "cassandra.reconnection.policy";
public static final String CASSANDRA_SPECULATIVE_EXECUTION_POLICY =
"cassandra.speculative.execution.policy";
"cassandra.speculative.execution.policy";
public static final String CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS =
"cassandra.max.schema.agreement.wait.second";
"cassandra.max.schema.agreement.wait.second";
public static final String CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_LOCAL =
"cassandra.pooling.new.connection.threshold.local";
"cassandra.pooling.new.connection.threshold.local";
public static final String CASSANDRA_POOLING_NEW_CONNECTION_THRESHOLD_REMOTE =
"cassandra.pooling.new.connection.threshold.remote";
"cassandra.pooling.new.connection.threshold.remote";
public static final String CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_LOCAL =
"cassandra.pooling.max.connection.per.host.local";
"cassandra.pooling.max.connection.per.host.local";
public static final String CASSANDRA_POOLING_MAX_CONNECTION_PER_HOST_REMOTE =
"cassandra.pooling.max.connection.per.host.remote";
"cassandra.pooling.max.connection.per.host.remote";
public static final String CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_LOCAL =
"cassandra.pooling.core.connection.per.host.local";
"cassandra.pooling.core.connection.per.host.local";
public static final String CASSANDRA_POOLING_CORE_CONNECTION_PER_HOST_REMOTE =
"cassandra.pooling.core.connection.per.host.remote";
"cassandra.pooling.core.connection.per.host.remote";
public static final String CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_LOCAL =
"cassandra.pooling.max.request.per.connection.local";
"cassandra.pooling.max.request.per.connection.local";
public static final String CASSANDRA_POOLING_MAX_REQUESTS_PER_CONNECTION_REMOTE =
"cassandra.pooling.max.request.per.connection.remote";
"cassandra.pooling.max.request.per.connection.remote";
public static final String CASSANDRA_POOLING_IDLE_TIMEOUT_SECONDS =
"cassandra.pooling.idle.timeout.seconds";
"cassandra.pooling.idle.timeout.seconds";
public static final String CASSANDRA_POOLING_POOL_TIMEOUT_MILLIS =
"cassandra.pooling.pool.timeout.millisecs";
"cassandra.pooling.pool.timeout.millisecs";
public static final String CASSANDRA_POOLING_HEARTBEAT_INTERVAL_SECONDS =
"cassandra.pooling.heartbeat.interval.seconds";
"cassandra.pooling.heartbeat.interval.seconds";
public static final String CASSANDRA_QUERY_DEFAULT_CONSISTENCY =
"cassandra.query.default.consistency";
"cassandra.query.default.consistency";
public static final String CASSANDRA_QUERY_DEFAULT_SERIAL_CONSISTENCY =
"cassandra.query.default.serial.consistency";
"cassandra.query.default.serial.consistency";
public static final String CASSANDRA_QUERY_DEFAULT_FETCH_SIZE =
"cassandra.query.default.fetchSize";
"cassandra.query.default.fetchSize";
public static final String CASSANDRA_QUERY_DEFAULT_IDEMPOTENCE =
"cassandra.query.default.idempotence";
"cassandra.query.default.idempotence";
public static final String CASSANDRA_SOCKET_CONNECTION_TIMEOUT_MILLIS =
"cassandra.socket.connection.timeout.millisecs";
public static final String CASSANDRA_SOCKET_KEEP_ALIVE =
"cassandra.socket.keep.alive";
"cassandra.socket.connection.timeout.millisecs";
public static final String CASSANDRA_SOCKET_KEEP_ALIVE = "cassandra.socket.keep.alive";
public static final String CASSANDRA_SOCKET_READ_TIMEOUT_MILLIS =
"cassandra.socket.read.timeout.millisecs";
"cassandra.socket.read.timeout.millisecs";
public static final String CASSANDRA_SOCKET_RECEIVED_BUFFER_SIZE_BYTES =
"cassandra.socket.received.buffer.size.bytes";
public static final String CASSANDRA_SOCKET_REUSE_ADDRESS =
"cassandra.socket.reuse.address";
"cassandra.socket.received.buffer.size.bytes";
public static final String CASSANDRA_SOCKET_REUSE_ADDRESS = "cassandra.socket.reuse.address";
public static final String CASSANDRA_SOCKET_SEND_BUFFER_SIZE_BYTES =
"cassandra.socket.send.buffer.size.bytes";
public static final String CASSANDRA_SOCKET_SO_LINGER =
"cassandra.socket.soLinger";
public static final String CASSANDRA_SOCKET_TCP_NO_DELAY =
"cassandra.socket.tcp.no_delay";
public static final String CASSANDRA_WITH_SSL =
"cassandra.ssl.enabled";
public static final String CASSANDRA_TRUSTSTORE_PATH =
"cassandra.ssl.truststore.path";
public static final String CASSANDRA_TRUSTSTORE_PASSWORD =
"cassandra.ssl.truststore.password";
"cassandra.socket.send.buffer.size.bytes";
public static final String CASSANDRA_SOCKET_SO_LINGER = "cassandra.socket.soLinger";
public static final String CASSANDRA_SOCKET_TCP_NO_DELAY = "cassandra.socket.tcp.no_delay";
public static final String CASSANDRA_WITH_SSL = "cassandra.ssl.enabled";
public static final String CASSANDRA_TRUSTSTORE_PATH = "cassandra.ssl.truststore.path";
public static final String CASSANDRA_TRUSTSTORE_PASSWORD = "cassandra.ssl.truststore.password";
public static final String DEFAULT_HOST = "localhost";
public static final String DEFAULT_PORT = "9042";
@ -177,25 +163,30 @@ public class CassandraInterpreter extends Interpreter {
hosts.append(address).append(",");
}
LOGGER.info("Bootstrapping Cassandra Java Driver to connect to " + hosts.toString() +
"on port " + port);
LOGGER.info(
"Bootstrapping Cassandra Java Driver to connect to "
+ hosts.toString()
+ "on port "
+ port);
Compression compression = driverConfig.getCompressionProtocol(this);
clusterBuilder = Cluster.builder()
clusterBuilder =
Cluster.builder()
.addContactPoints(addresses)
.withPort(port)
.withProtocolVersion(driverConfig.getProtocolVersion(this))
.withClusterName(getProperty(CASSANDRA_CLUSTER_NAME))
.withCompression(compression)
.withCredentials(getProperty(CASSANDRA_CREDENTIALS_USERNAME),
getProperty(CASSANDRA_CREDENTIALS_PASSWORD))
.withCredentials(
getProperty(CASSANDRA_CREDENTIALS_USERNAME),
getProperty(CASSANDRA_CREDENTIALS_PASSWORD))
.withLoadBalancingPolicy(driverConfig.getLoadBalancingPolicy(this))
.withRetryPolicy(driverConfig.getRetryPolicy(this))
.withReconnectionPolicy(driverConfig.getReconnectionPolicy(this))
.withSpeculativeExecutionPolicy(driverConfig.getSpeculativeExecutionPolicy(this))
.withMaxSchemaAgreementWaitSeconds(
parseInt(getProperty(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS)))
parseInt(getProperty(CASSANDRA_MAX_SCHEMA_AGREEMENT_WAIT_SECONDS)))
.withPoolingOptions(driverConfig.getPoolingOptions(this))
.withQueryOptions(driverConfig.getQueryOptions(this))
.withSocketOptions(driverConfig.getSocketOptions(this));
@ -208,20 +199,19 @@ public class CassandraInterpreter extends Interpreter {
final SSLContext sslContext;
{
final KeyStore trustStore = KeyStore.getInstance("JKS");
final InputStream stream = Files.newInputStream(Paths.get(
getProperty(CASSANDRA_TRUSTSTORE_PATH)));
final InputStream stream =
Files.newInputStream(Paths.get(getProperty(CASSANDRA_TRUSTSTORE_PATH)));
trustStore.load(stream, getProperty(CASSANDRA_TRUSTSTORE_PASSWORD).toCharArray());
final TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(
TrustManagerFactory.getDefaultAlgorithm());
final TrustManagerFactory trustManagerFactory =
TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
trustManagerFactory.init(trustStore);
sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, trustManagerFactory.getTrustManagers(), null);
}
clusterBuilder = clusterBuilder.withSSL(JdkSSLOptions.builder()
.withSSLContext(sslContext)
.build());
clusterBuilder =
clusterBuilder.withSSL(JdkSSLOptions.builder().withSSLContext(sslContext).build());
} catch (Exception e) {
LOGGER.error(e.toString());
}
@ -246,8 +236,7 @@ public class CassandraInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {
}
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
@ -260,15 +249,16 @@ public class CassandraInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return NO_COMPLETION;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton()
.createOrGetParallelScheduler(CassandraInterpreter.class.getName() + this.hashCode(),
parseInt(getProperty(CASSANDRA_INTERPRETER_PARALLELISM)));
.createOrGetParallelScheduler(
CassandraInterpreter.class.getName() + this.hashCode(),
parseInt(getProperty(CASSANDRA_INTERPRETER_PARALLELISM)));
}
}

6
cassandra/src/main/java/org/apache/zeppelin/cassandra/ParsingException.java
View file

@ -16,10 +16,8 @@
*/
package org.apache.zeppelin.cassandra;
/**
* Parsing Exception for Cassandra CQL statement.
*/
public class ParsingException extends RuntimeException{
/** Parsing Exception for Cassandra CQL statement. */
public class ParsingException extends RuntimeException {
public ParsingException(String message) {
super(message);
}

640
cassandra/src/test/java/org/apache/zeppelin/cassandra/CassandraInterpreterTest.java
View file

File diff suppressed because it is too large Load diff

230
cassandra/src/test/java/org/apache/zeppelin/cassandra/InterpreterLogicTest.java
View file

@ -16,6 +16,13 @@
*/
package org.apache.zeppelin.cassandra;
import static com.datastax.driver.core.BatchStatement.Type.UNLOGGED;
import static com.datastax.driver.core.ConsistencyLevel.ALL;
import static com.datastax.driver.core.ConsistencyLevel.LOCAL_SERIAL;
import static com.datastax.driver.core.ConsistencyLevel.ONE;
import static com.datastax.driver.core.ConsistencyLevel.QUORUM;
import static com.datastax.driver.core.ConsistencyLevel.SERIAL;
import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
@ -24,39 +31,16 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import static java.util.Arrays.asList;
import static com.datastax.driver.core.BatchStatement.Type.UNLOGGED;
import static com.datastax.driver.core.ConsistencyLevel.ALL;
import static com.datastax.driver.core.ConsistencyLevel.LOCAL_SERIAL;
import static com.datastax.driver.core.ConsistencyLevel.ONE;
import static com.datastax.driver.core.ConsistencyLevel.QUORUM;
import static com.datastax.driver.core.ConsistencyLevel.SERIAL;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.Statement;
import scala.Option;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.AnyBlock;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.Consistency;
import org.apache.zeppelin.cassandra.TextBlockHierarchy.DowngradingRetryPolicy$;
@ -72,80 +56,90 @@ import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.mockito.Answers;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.runners.MockitoJUnitRunner;
import scala.Option;
@RunWith(MockitoJUnitRunner.class)
public class InterpreterLogicTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Rule public ExpectedException expectedException = ExpectedException.none();
@Mock(answer = Answers.RETURNS_DEEP_STUBS)
private InterpreterContext intrContext;
@Mock
private Session session;
@Mock private Session session;
final InterpreterLogic helper = new InterpreterLogic(session);
@Captor
ArgumentCaptor<ParamOption[]> optionsCaptor;
@Captor ArgumentCaptor<ParamOption[]> optionsCaptor;
@Test
public void should_parse_input_string_block() throws Exception {
//Given
// Given
String input = "SELECT * FROM users LIMIT 10;";
//When
// When
final List<AnyBlock> anyBlocks = this.<AnyBlock>toJavaList(helper.parseInput(input));
//Then
// Then
assertThat(anyBlocks).hasSize(1);
assertThat(anyBlocks.get(0)).isInstanceOf(SimpleStm.class);
}
@Test
public void should_exception_while_parsing_input() throws Exception {
//Given
// Given
String input = "SELECT * FROM users LIMIT 10";
//When
// When
expectedException.expect(InterpreterException.class);
expectedException.expectMessage("Error parsing input:\n" +
"\t'SELECT * FROM users LIMIT 10'\n" +
"Did you forget to add ; (semi-colon) at the end of each CQL statement ?");
expectedException.expectMessage(
"Error parsing input:\n"
+ "\t'SELECT * FROM users LIMIT 10'\n"
+ "Did you forget to add ; (semi-colon) at the end of each CQL statement ?");
helper.parseInput(input);
}
@Test
public void should_extract_variable_and_default_value() throws Exception {
//Given
// Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getGui().input("table", "zeppelin.demo")).thenReturn("zeppelin.demo");
when(intrContext.getGui().input("id", "'John'")).thenReturn("'John'");
//When
final String actual = helper.maybeExtractVariables(
// When
final String actual =
helper.maybeExtractVariables(
"SELECT * FROM {{table=zeppelin.demo}} WHERE id={{id='John'}}", intrContext);
//Then
// Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='John'");
}
@Test
public void should_extract_variable_and_choices() throws Exception {
//Given
// Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getGui().select(eq("name"), eq("'Paul'"), optionsCaptor.capture()))
.thenReturn("'Jack'");
.thenReturn("'Jack'");
//When
final String actual = helper.maybeExtractVariables(
// When
final String actual =
helper.maybeExtractVariables(
"SELECT * FROM zeppelin.artists WHERE name={{name='Paul'|'Jack'|'Smith'}}",
intrContext);
//Then
// Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.artists WHERE name='Jack'");
final List<ParamOption> paramOptions = asList(optionsCaptor.getValue());
assertThat(paramOptions.get(0).getValue()).isEqualTo("'Paul'");
@ -155,131 +149,137 @@ public class InterpreterLogicTest {
@Test
public void should_extract_no_variable() throws Exception {
//Given
// Given
GUI gui = mock(GUI.class);
when(intrContext.getGui()).thenReturn(gui);
//When
// When
final String actual = helper.maybeExtractVariables("SELECT * FROM zeppelin.demo", intrContext);
//Then
// Then
verifyZeroInteractions(gui);
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo");
}
@Test
public void should_extract_variable_from_angular_object_registry() throws Exception {
//Given
// Given
AngularObjectRegistry angularObjectRegistry = new AngularObjectRegistry("cassandra", null);
angularObjectRegistry.add("id", "from_angular_registry", "noteId", "paragraphId");
when(intrContext.getAngularObjectRegistry()).thenReturn(angularObjectRegistry);
when(intrContext.getNoteId()).thenReturn("noteId");
when(intrContext.getParagraphId()).thenReturn("paragraphId");
//When
final String actual = helper.maybeExtractVariables(
// When
final String actual =
helper.maybeExtractVariables(
"SELECT * FROM zeppelin.demo WHERE id='{{id=John}}'", intrContext);
//Then
// Then
assertThat(actual).isEqualTo("SELECT * FROM zeppelin.demo WHERE id='from_angular_registry'");
verify(intrContext, never()).getGui();
}
@Test
public void should_error_if_incorrect_variable_definition() throws Exception {
//Given
// Given
//When
// When
expectedException.expect(ParsingException.class);
expectedException.expectMessage("Invalid bound variable definition for " +
"'{{table?zeppelin.demo}}' in 'SELECT * FROM {{table?zeppelin.demo}} " +
"WHERE id={{id='John'}}'. It should be of form 'variable=defaultValue'");
expectedException.expectMessage(
"Invalid bound variable definition for "
+ "'{{table?zeppelin.demo}}' in 'SELECT * FROM {{table?zeppelin.demo}} "
+ "WHERE id={{id='John'}}'. It should be of form 'variable=defaultValue'");
//Then
helper.maybeExtractVariables("SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}",
intrContext);
// Then
helper.maybeExtractVariables(
"SELECT * FROM {{table?zeppelin.demo}} WHERE id={{id='John'}}", intrContext);
}
@Test
public void should_extract_consistency_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new Consistency(ALL),
new Consistency(ONE));
// Given
List<QueryParameters> options =
Arrays.<QueryParameters>asList(new Consistency(ALL), new Consistency(ONE));
//When
// When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
// Then
assertThat(actual.consistency().get()).isEqualTo(ALL);
}
@Test
public void should_extract_serial_consistency_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new SerialConsistency(SERIAL),
new SerialConsistency(LOCAL_SERIAL));
// Given
List<QueryParameters> options =
Arrays.<QueryParameters>asList(
new SerialConsistency(SERIAL), new SerialConsistency(LOCAL_SERIAL));
//When
// When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
// Then
assertThat(actual.serialConsistency().get()).isEqualTo(SERIAL);
}
@Test
public void should_extract_timestamp_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new Timestamp(123L),
new Timestamp(456L));
// Given
List<QueryParameters> options =
Arrays.<QueryParameters>asList(new Timestamp(123L), new Timestamp(456L));
//When
// When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
// Then
assertThat(actual.timestamp().get()).isEqualTo(123L);
}
@Test
public void should_extract_retry_policy_option() throws Exception {
//Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(DowngradingRetryPolicy$.MODULE$,
LoggingDefaultRetryPolicy$.MODULE$);
// Given
List<QueryParameters> options =
Arrays.<QueryParameters>asList(
DowngradingRetryPolicy$.MODULE$, LoggingDefaultRetryPolicy$.MODULE$);
//When
// When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
// Then
assertThat(actual.retryPolicy().get()).isSameAs(DowngradingRetryPolicy$.MODULE$);
}
@Test
public void should_extract_request_timeout_option() throws Exception {
//Given
// Given
List<QueryParameters> options = Arrays.<QueryParameters>asList(new RequestTimeOut(100));
//When
// When
final CassandraQueryOptions actual = helper.extractQueryOptions(toScalaList(options));
//Then
// Then
assertThat(actual.requestTimeOut().get()).isEqualTo(100);
}
@Test
public void should_generate_simple_statement() throws Exception {
//Given
// Given
String input = "SELECT * FROM users LIMIT 10;";
CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM),
CassandraQueryOptions options =
new CassandraQueryOptions(
Option.apply(QUORUM),
Option.<ConsistencyLevel>empty(),
Option.empty(),
Option.<RetryPolicy>empty(),
Option.empty(),
Option.empty());
//When
final SimpleStatement actual = helper.generateSimpleStatement(new SimpleStm(input), options,
intrContext);
// When
final SimpleStatement actual =
helper.generateSimpleStatement(new SimpleStm(input), options, intrContext);
//Then
// Then
assertThat(actual).isNotNull();
assertThat(actual.getQueryString()).isEqualTo("SELECT * FROM users LIMIT 10;");
assertThat(actual.getConsistencyLevel()).isSameAs(QUORUM);
@ -287,22 +287,24 @@ public class InterpreterLogicTest {
@Test
public void should_generate_batch_statement() throws Exception {
//Given
// Given
Statement st1 = new SimpleStatement("SELECT * FROM users LIMIT 10;");
Statement st2 = new SimpleStatement("INSERT INTO users(id) VALUES(10);");
Statement st3 = new SimpleStatement("UPDATE users SET name = 'John DOE' WHERE id=10;");
CassandraQueryOptions options = new CassandraQueryOptions(Option.apply(QUORUM),
CassandraQueryOptions options =
new CassandraQueryOptions(
Option.apply(QUORUM),
Option.<ConsistencyLevel>empty(),
Option.empty(),
Option.<RetryPolicy>empty(),
Option.empty(),
Option.empty());
//When
BatchStatement actual = helper.generateBatchStatement(UNLOGGED, options,
toScalaList(asList(st1, st2, st3)));
// When
BatchStatement actual =
helper.generateBatchStatement(UNLOGGED, options, toScalaList(asList(st1, st2, st3)));
//Then
// Then
assertThat(actual).isNotNull();
final List<Statement> statements = new ArrayList<>(actual.getStatements());
assertThat(statements).hasSize(3);
@ -314,26 +316,26 @@ public class InterpreterLogicTest {
@Test
public void should_parse_bound_values() throws Exception {
//Given
// Given
String bs = "'jdoe',32,'John DOE',null, true, '2014-06-12 34:00:34'";
//When
// When
final List<String> actual = this.<String>toJavaList(helper.parseBoundValues("ps", bs));
//Then
assertThat(actual).containsExactly("'jdoe'", "32", "'John DOE'",
"null", "true", "2014-06-12 34:00:34");
// Then
assertThat(actual)
.containsExactly("'jdoe'", "32", "'John DOE'", "null", "true", "2014-06-12 34:00:34");
}
@Test
public void should_parse_simple_date() throws Exception {
//Given
// Given
String dateString = "2015-07-30 12:00:01";
//When
// When
final Date actual = helper.parseDate(dateString);
//Then
// Then
Calendar calendar = Calendar.getInstance();
calendar.setTime(actual);
@ -347,13 +349,13 @@ public class InterpreterLogicTest {
@Test
public void should_parse_accurate_date() throws Exception {
//Given
// Given
String dateString = "2015-07-30 12:00:01.123";
//When
// When
final Date actual = helper.parseDate(dateString);
//Then
// Then
Calendar calendar = Calendar.getInstance();
calendar.setTime(actual);
@ -366,11 +368,11 @@ public class InterpreterLogicTest {
assertThat(calendar.get(Calendar.MILLISECOND)).isEqualTo(123);
}
private <A> scala.collection.immutable.List<A> toScalaList(java.util.List<A> list) {
private <A> scala.collection.immutable.List<A> toScalaList(java.util.List<A> list) {
return scala.collection.JavaConversions.collectionAsScalaIterable(list).toList();
}
private <A> java.util.List<A> toJavaList(scala.collection.immutable.List<A> list){
private <A> java.util.List<A> toJavaList(scala.collection.immutable.List<A> list) {
return scala.collection.JavaConversions.seqAsJavaList(list);
}
}

4
docs/development/contribution/how_to_contribute_code.md
View file

@ -37,6 +37,10 @@ Since Zeppelin uses Git for it's SCM system, you need git client installed in yo
You are free to use whatever IDE you prefer, or your favorite command line editor.
#### Code Style
We decided to use `google-java-format`. You can install this formatter from [https://github.com/google/google-java-format](https://github.com/google/google-java-format). And the build script also contains `fmt-maven-plugin` to verify the formats of files. You can check `mvn validate` and fix wrong formats by running `mvn fmt:format` manually.
#### Build Tools
To build the code, install

7
elasticsearch/pom.xml
View file

@ -101,13 +101,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

176
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreter.java
View file

@ -17,23 +17,10 @@
package org.apache.zeppelin.elasticsearch;
import com.github.wnameless.json.flattener.JsonFlattener;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -48,9 +35,7 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.github.wnameless.json.flattener.JsonFlattener;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.elasticsearch.action.ActionResponse;
import org.apache.zeppelin.elasticsearch.action.AggWrapper;
@ -62,33 +47,43 @@ import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Elasticsearch Interpreter for Zeppelin.
*/
/** Elasticsearch Interpreter for Zeppelin. */
public class ElasticsearchInterpreter extends Interpreter {
private static Logger logger = LoggerFactory.getLogger(ElasticsearchInterpreter.class);
private static final String HELP = "Elasticsearch interpreter:\n"
+ "General format: <command> /<indices>/<types>/<id> <option> <JSON>\n"
+ " - indices: list of indices separated by commas (depends on the command)\n"
+ " - types: list of document types separated by commas (depends on the command)\n"
+ "Commands:\n"
+ " - search /indices/types <query>\n"
+ " . indices and types can be omitted (at least, you have to provide '/')\n"
+ " . a query is either a JSON-formatted query, nor a lucene query\n"
+ " - size <value>\n"
+ " . defines the size of the result set (default value is in the config)\n"
+ " . if used, this command must be declared before a search command\n"
+ " - count /indices/types <query>\n"
+ " . same comments as for the search\n"
+ " - get /index/type/id\n"
+ " - delete /index/type/id\n"
+ " - index /index/type/id <json-formatted document>\n"
+ " . the id can be omitted, elasticsearch will generate one";
private static final String HELP =
"Elasticsearch interpreter:\n"
+ "General format: <command> /<indices>/<types>/<id> <option> <JSON>\n"
+ " - indices: list of indices separated by commas (depends on the command)\n"
+ " - types: list of document types separated by commas (depends on the command)\n"
+ "Commands:\n"
+ " - search /indices/types <query>\n"
+ " . indices and types can be omitted (at least, you have to provide '/')\n"
+ " . a query is either a JSON-formatted query, nor a lucene query\n"
+ " - size <value>\n"
+ " . defines the size of the result set (default value is in the config)\n"
+ " . if used, this command must be declared before a search command\n"
+ " - count /indices/types <query>\n"
+ " . same comments as for the search\n"
+ " - get /index/type/id\n"
+ " - delete /index/type/id\n"
+ " - index /index/type/id <json-formatted document>\n"
+ " . the id can be omitted, elasticsearch will generate one";
protected static final List<String> COMMANDS = Arrays.asList(
"count", "delete", "get", "help", "index", "search");
protected static final List<String> COMMANDS =
Arrays.asList("count", "delete", "get", "help", "index", "search");
private static final Pattern FIELD_NAME_PATTERN = Pattern.compile("\\[\\\\\"(.+)\\\\\"\\](.*)");
@ -119,8 +114,12 @@ public class ElasticsearchInterpreter extends Interpreter {
this.resultSize = Integer.parseInt(getProperty(ELASTICSEARCH_RESULT_SIZE));
} catch (final NumberFormatException e) {
this.resultSize = 10;
logger.error("Unable to parse " + ELASTICSEARCH_RESULT_SIZE + " : " +
getProperty(ELASTICSEARCH_RESULT_SIZE), e);
logger.error(
"Unable to parse "
+ ELASTICSEARCH_RESULT_SIZE
+ " : "
+ getProperty(ELASTICSEARCH_RESULT_SIZE),
e);
}
try {
@ -154,8 +153,9 @@ public class ElasticsearchInterpreter extends Interpreter {
int currentResultSize = resultSize;
if (elsClient == null) {
return new InterpreterResult(InterpreterResult.Code.ERROR,
"Problem with the Elasticsearch client, please check your configuration (host, port,...)");
return new InterpreterResult(
InterpreterResult.Code.ERROR,
"Problem with the Elasticsearch client, please check your configuration (host, port,...)");
}
String[] items = StringUtils.split(cmd.trim(), " ", 3);
@ -171,8 +171,7 @@ public class ElasticsearchInterpreter extends Interpreter {
final String[] lines = StringUtils.split(cmd.trim(), "\n", 2);
if (lines.length < 2) {
return processHelp(InterpreterResult.Code.ERROR,
"Size cmd must be followed by a search");
return processHelp(InterpreterResult.Code.ERROR, "Size cmd must be followed by a search");
}
final String[] sizeLine = StringUtils.split(lines[0], " ", 2);
@ -229,8 +228,8 @@ public class ElasticsearchInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String s, int i,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String s, int i, InterpreterContext interpreterContext) {
final List suggestions = new ArrayList<>();
for (final String cmd : COMMANDS) {
@ -242,9 +241,9 @@ public class ElasticsearchInterpreter extends Interpreter {
}
private void addAngularObject(InterpreterContext interpreterContext, String prefix, Object obj) {
interpreterContext.getAngularObjectRegistry().add(
prefix + "_" + interpreterContext.getParagraphId().replace("-", "_"),
obj, null, null);
interpreterContext
.getAngularObjectRegistry()
.add(prefix + "_" + interpreterContext.getParagraphId().replace("-", "_"), obj, null, null);
}
private String[] getIndexTypeId(String[] urlItems) {
@ -256,13 +255,11 @@ public class ElasticsearchInterpreter extends Interpreter {
final String type = urlItems[1];
final String id = StringUtils.join(Arrays.copyOfRange(urlItems, 2, urlItems.length), '/');
if (StringUtils.isEmpty(index)
|| StringUtils.isEmpty(type)
|| StringUtils.isEmpty(id)) {
if (StringUtils.isEmpty(index) || StringUtils.isEmpty(type) || StringUtils.isEmpty(id)) {
return null;
}
return new String[] { index, type, id };
return new String[] {index, type, id};
}
private InterpreterResult processHelp(InterpreterResult.Code code, String additionalMessage) {
@ -287,8 +284,8 @@ public class ElasticsearchInterpreter extends Interpreter {
final String[] indexTypeId = getIndexTypeId(urlItems);
if (indexTypeId == null) {
return new InterpreterResult(InterpreterResult.Code.ERROR,
"Bad URL (it should be /index/type/id)");
return new InterpreterResult(
InterpreterResult.Code.ERROR, "Bad URL (it should be /index/type/id)");
}
final ActionResponse response = elsClient.get(indexTypeId[0], indexTypeId[1], indexTypeId[2]);
@ -300,9 +297,7 @@ public class ElasticsearchInterpreter extends Interpreter {
addAngularObject(interpreterContext, "get", json);
return new InterpreterResult(
InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.TEXT,
jsonStr);
InterpreterResult.Code.SUCCESS, InterpreterResult.Type.TEXT, jsonStr);
}
return new InterpreterResult(InterpreterResult.Code.ERROR, "Document not found");
@ -316,10 +311,11 @@ public class ElasticsearchInterpreter extends Interpreter {
* @param interpreterContext Instance of the context
* @return Result of the count request, it contains the total hits
*/
private InterpreterResult processCount(String[] urlItems, String data,
InterpreterContext interpreterContext) {
private InterpreterResult processCount(
String[] urlItems, String data, InterpreterContext interpreterContext) {
if (urlItems.length > 2) {
return new InterpreterResult(InterpreterResult.Code.ERROR,
return new InterpreterResult(
InterpreterResult.Code.ERROR,
"Bad URL (it should be /index1,index2,.../type1,type2,...)");
}
@ -328,9 +324,7 @@ public class ElasticsearchInterpreter extends Interpreter {
addAngularObject(interpreterContext, "count", response.getTotalHits());
return new InterpreterResult(
InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.TEXT,
"" + response.getTotalHits());
InterpreterResult.Code.SUCCESS, InterpreterResult.Type.TEXT, "" + response.getTotalHits());
}
/**
@ -342,18 +336,22 @@ public class ElasticsearchInterpreter extends Interpreter {
* @param interpreterContext Instance of the context
* @return Result of the search request, it contains a tab-formatted string of the matching hits
*/
private InterpreterResult processSearch(String[] urlItems, String data, int size,
InterpreterContext interpreterContext) {
private InterpreterResult processSearch(
String[] urlItems, String data, int size, InterpreterContext interpreterContext) {
if (urlItems.length > 2) {
return new InterpreterResult(InterpreterResult.Code.ERROR,
return new InterpreterResult(
InterpreterResult.Code.ERROR,
"Bad URL (it should be /index1,index2,.../type1,type2,...)");
}
final ActionResponse response = searchData(urlItems, data, size);
addAngularObject(interpreterContext, "search",
(response.getAggregations() != null && response.getAggregations().size() > 0) ?
response.getAggregations() : response.getHits());
addAngularObject(
interpreterContext,
"search",
(response.getAggregations() != null && response.getAggregations().size() > 0)
? response.getAggregations()
: response.getHits());
return buildResponseMessage(response);
}
@ -367,17 +365,15 @@ public class ElasticsearchInterpreter extends Interpreter {
*/
private InterpreterResult processIndex(String[] urlItems, String data) {
if (urlItems.length < 2 || urlItems.length > 3) {
return new InterpreterResult(InterpreterResult.Code.ERROR,
"Bad URL (it should be /index/type or /index/type/id)");
return new InterpreterResult(
InterpreterResult.Code.ERROR, "Bad URL (it should be /index/type or /index/type/id)");
}
final ActionResponse response = elsClient.index(
urlItems[0], urlItems[1], urlItems.length == 2 ? null : urlItems[2], data);
final ActionResponse response =
elsClient.index(urlItems[0], urlItems[1], urlItems.length == 2 ? null : urlItems[2], data);
return new InterpreterResult(
InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.TEXT,
response.getHit().getId());
InterpreterResult.Code.SUCCESS, InterpreterResult.Type.TEXT, response.getHit().getId());
}
/**
@ -390,8 +386,8 @@ public class ElasticsearchInterpreter extends Interpreter {
final String[] indexTypeId = getIndexTypeId(urlItems);
if (indexTypeId == null) {
return new InterpreterResult(InterpreterResult.Code.ERROR,
"Bad URL (it should be /index/type/id)");
return new InterpreterResult(
InterpreterResult.Code.ERROR, "Bad URL (it should be /index/type/id)");
}
final ActionResponse response =
@ -399,9 +395,7 @@ public class ElasticsearchInterpreter extends Interpreter {
if (response.isSucceeded()) {
return new InterpreterResult(
InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.TEXT,
response.getHit().getId());
InterpreterResult.Code.SUCCESS, InterpreterResult.Type.TEXT, response.getHit().getId());
}
return new InterpreterResult(InterpreterResult.Code.ERROR, "Document not found");
@ -451,7 +445,7 @@ public class ElasticsearchInterpreter extends Interpreter {
final StringBuffer buffer = new StringBuffer();
final String[] keys = headerKeys.toArray(new String[0]);
for (final String key: keys) {
for (final String key : keys) {
buffer.append("\t" + key);
}
buffer.deleteCharAt(0);
@ -459,7 +453,7 @@ public class ElasticsearchInterpreter extends Interpreter {
for (final Map<String, Object> bucket : buckets) {
buffer.append("\n");
for (final String key: keys) {
for (final String key : keys) {
buffer.append(bucket.get(key)).append("\t");
}
buffer.deleteCharAt(buffer.length() - 1);
@ -479,7 +473,7 @@ public class ElasticsearchInterpreter extends Interpreter {
final Set<String> headerKeys = new HashSet<>();
final List<Map<String, Object>> buckets = new LinkedList<>();
for (final AggWrapper aggregation: aggregations) {
for (final AggWrapper aggregation : aggregations) {
final Map<String, Object> bucketMap = JsonFlattener.flattenAsMap(aggregation.getResult());
headerKeys.addAll(bucketMap.keySet());
buckets.add(bucketMap);
@ -487,7 +481,7 @@ public class ElasticsearchInterpreter extends Interpreter {
final StringBuffer buffer = new StringBuffer();
final String[] keys = headerKeys.toArray(new String[0]);
for (final String key: keys) {
for (final String key : keys) {
buffer.append("\t" + key);
}
buffer.deleteCharAt(0);
@ -495,7 +489,7 @@ public class ElasticsearchInterpreter extends Interpreter {
for (final Map<String, Object> bucket : buckets) {
buffer.append("\n");
for (final String key: keys) {
for (final String key : keys) {
buffer.append(bucket.get(key)).append("\t");
}
buffer.deleteCharAt(buffer.length() - 1);
@ -511,7 +505,7 @@ public class ElasticsearchInterpreter extends Interpreter {
return "";
}
//First : get all the keys in order to build an ordered list of the values for each hit
// First : get all the keys in order to build an ordered list of the values for each hit
//
final List<Map<String, Object>> flattenHits = new LinkedList<>();
final Set<String> keys = new TreeSet<>();
@ -526,8 +520,8 @@ public class ElasticsearchInterpreter extends Interpreter {
final String fieldName = iter.next();
final Matcher fieldNameMatcher = FIELD_NAME_PATTERN.matcher(fieldName);
if (fieldNameMatcher.matches()) {
flattenMap.put(fieldNameMatcher.group(1) + fieldNameMatcher.group(2),
flattenJsonMap.get(fieldName));
flattenMap.put(
fieldNameMatcher.group(1) + fieldNameMatcher.group(2), flattenJsonMap.get(fieldName));
} else {
flattenMap.put(fieldName, flattenJsonMap.get(fieldName));
}

4
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/action/ActionException.java
View file

@ -17,9 +17,7 @@
package org.apache.zeppelin.elasticsearch.action;
/**
* Runtime exception thrown when there is a problem during an action (search, get, ...).
*/
/** Runtime exception thrown when there is a problem during an action (search, get, ...). */
public class ActionException extends RuntimeException {
public ActionException(String message) {

5
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/action/ActionResponse.java
View file

@ -20,9 +20,7 @@ package org.apache.zeppelin.elasticsearch.action;
import java.util.LinkedList;
import java.util.List;
/**
* Contains the result of an action (hits, aggregations, ...).
*/
/** Contains the result of an action (hits, aggregations, ...). */
public class ActionResponse {
private boolean succeeded;
@ -30,7 +28,6 @@ public class ActionResponse {
private final List<HitWrapper> hits = new LinkedList<>();
private final List<AggWrapper> aggregations = new LinkedList<>();
public ActionResponse succeeded(boolean succeeded) {
this.succeeded = succeeded;
return this;

13
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/action/AggWrapper.java
View file

@ -17,14 +17,13 @@
package org.apache.zeppelin.elasticsearch.action;
/**
* Contains the result of an aggregation.
*/
/** Contains the result of an aggregation. */
public class AggWrapper {
/**
* Type of an aggregation (to know if there are buckets or not).
*/
public enum AggregationType { SIMPLE, MULTI_BUCKETS };
/** Type of an aggregation (to know if there are buckets or not). */
public enum AggregationType {
SIMPLE,
MULTI_BUCKETS
};
private final AggregationType type;
private final String result;

4
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/action/HitWrapper.java
View file

@ -21,9 +21,7 @@ import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
/**
* Contains the data of a hit.
*/
/** Contains the data of a hit. */
public class HitWrapper {
private final JsonParser parser = new JsonParser();

4
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/ElasticsearchClient.java
View file

@ -19,9 +19,7 @@ package org.apache.zeppelin.elasticsearch.client;
import org.apache.zeppelin.elasticsearch.action.ActionResponse;
/**
* Interface that must be implemented by any kind of Elasticsearch client (transport, ...).
*/
/** Interface that must be implemented by any kind of Elasticsearch client (transport, ...). */
public interface ElasticsearchClient {
ActionResponse get(String index, String type, String id);

136
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/HttpBasedClient.java
View file

@ -21,34 +21,28 @@ import com.google.common.base.Joiner;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonParseException;
import org.apache.commons.lang3.StringUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.JsonNode;
import com.mashape.unirest.http.Unirest;
import com.mashape.unirest.http.exceptions.UnirestException;
import com.mashape.unirest.request.HttpRequest;
import com.mashape.unirest.request.HttpRequestWithBody;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter;
import org.apache.zeppelin.elasticsearch.action.ActionException;
import org.apache.zeppelin.elasticsearch.action.ActionResponse;
import org.apache.zeppelin.elasticsearch.action.AggWrapper;
import org.apache.zeppelin.elasticsearch.action.AggWrapper.AggregationType;
import org.apache.zeppelin.elasticsearch.action.HitWrapper;
import org.json.JSONArray;
import org.json.JSONObject;
/**
* Elasticsearch client using the HTTP API.
*/
/** Elasticsearch client using the HTTP API. */
public class HttpBasedClient implements ElasticsearchClient {
private static final String QUERY_STRING_TEMPLATE =
"{ \"query\": { \"query_string\": { \"query\": \"_Q_\", \"analyze_wildcard\": \"true\" } } }";
@ -118,8 +112,11 @@ public class HttpBasedClient implements ElasticsearchClient {
} else {
// There are differences: to avoid problems with some special characters
// such as / and # in id, use a "terms" query
buffer.append("/_search?source=").append(URLEncoder
.encode("{\"query\":{\"terms\":{\"_id\":[\"" + id + "\"]}}}", "UTF-8"));
buffer
.append("/_search?source=")
.append(
URLEncoder.encode(
"{\"query\":{\"terms\":{\"_id\":[\"" + id + "\"]}}}", "UTF-8"));
}
} else {
buffer.append("/").append(id);
@ -154,28 +151,31 @@ public class HttpBasedClient implements ElasticsearchClient {
if (isSucceeded) {
final JsonNode body = new JsonNode(result.getBody());
if (body.getObject().has("_index")) {
response = new ActionResponse()
.succeeded(true)
.hit(new HitWrapper(
getFieldAsString(body, "_index"),
getFieldAsString(body, "_type"),
getFieldAsString(body, "_id"),
getFieldAsString(body, "_source")));
response =
new ActionResponse()
.succeeded(true)
.hit(
new HitWrapper(
getFieldAsString(body, "_index"),
getFieldAsString(body, "_type"),
getFieldAsString(body, "_id"),
getFieldAsString(body, "_source")));
} else {
final JSONArray hits = getFieldAsArray(body.getObject(), "hits/hits");
final JSONObject hit = (JSONObject) hits.iterator().next();
response = new ActionResponse()
.succeeded(true)
.hit(new HitWrapper(
hit.getString("_index"),
hit.getString("_type"),
hit.getString("_id"),
hit.opt("_source").toString()));
response =
new ActionResponse()
.succeeded(true)
.hit(
new HitWrapper(
hit.getString("_index"),
hit.getString("_type"),
hit.getString("_id"),
hit.opt("_source").toString()));
}
} else {
if (result.getStatus() == 404) {
response = new ActionResponse()
.succeeded(false);
response = new ActionResponse().succeeded(false);
} else {
throw new ActionException(result.getBody());
}
@ -200,13 +200,15 @@ public class HttpBasedClient implements ElasticsearchClient {
if (isSucceeded) {
final JsonNode body = new JsonNode(result.getBody());
response = new ActionResponse()
.succeeded(true)
.hit(new HitWrapper(
getFieldAsString(body, "_index"),
getFieldAsString(body, "_type"),
getFieldAsString(body, "_id"),
null));
response =
new ActionResponse()
.succeeded(true)
.hit(
new HitWrapper(
getFieldAsString(body, "_index"),
getFieldAsString(body, "_type"),
getFieldAsString(body, "_id"),
null));
} else {
throw new ActionException(result.getBody());
}
@ -229,7 +231,8 @@ public class HttpBasedClient implements ElasticsearchClient {
request
.header("Accept", "application/json")
.header("Content-Type", "application/json")
.body(data).getHttpRequest();
.body(data)
.getHttpRequest();
if (StringUtils.isNotEmpty(username)) {
request.basicAuth(username, password);
}
@ -238,13 +241,15 @@ public class HttpBasedClient implements ElasticsearchClient {
final boolean isSucceeded = isSucceeded(result);
if (isSucceeded) {
response = new ActionResponse()
.succeeded(true)
.hit(new HitWrapper(
getFieldAsString(result, "_index"),
getFieldAsString(result, "_type"),
getFieldAsString(result, "_id"),
null));
response =
new ActionResponse()
.succeeded(true)
.hit(
new HitWrapper(
getFieldAsString(result, "_index"),
getFieldAsString(result, "_type"),
getFieldAsString(result, "_id"),
null));
} else {
throw new ActionException(result.getBody().toString());
}
@ -270,9 +275,9 @@ public class HttpBasedClient implements ElasticsearchClient {
}
try {
final HttpRequestWithBody request = Unirest
.post(getUrl(indices, types) + "/_search?size=" + size)
.header("Content-Type", "application/json");
final HttpRequestWithBody request =
Unirest.post(getUrl(indices, types) + "/_search?size=" + size)
.header("Content-Type", "application/json");
if (StringUtils.isNoneEmpty(query)) {
request.header("Accept", "application/json").body(query);
@ -288,9 +293,7 @@ public class HttpBasedClient implements ElasticsearchClient {
if (isSucceeded(result)) {
final long total = getFieldAsLong(result, "hits/total");
response = new ActionResponse()
.succeeded(true)
.totalHits(total);
response = new ActionResponse().succeeded(true).totalHits(total);
if (containsAggs(result)) {
JSONObject aggregationsMap = body.getJSONObject("aggregations");
@ -298,7 +301,7 @@ public class HttpBasedClient implements ElasticsearchClient {
aggregationsMap = body.getJSONObject("aggs");
}
for (final String key: aggregationsMap.keySet()) {
for (final String key : aggregationsMap.keySet()) {
final JSONObject aggResult = aggregationsMap.getJSONObject(key);
if (aggResult.has("buckets")) {
// Multi-bucket aggregations
@ -319,13 +322,13 @@ public class HttpBasedClient implements ElasticsearchClient {
while (iter.hasNext()) {
final JSONObject hit = (JSONObject) iter.next();
final Object data =
hit.opt("_source") != null ? hit.opt("_source") : hit.opt("fields");
response.addHit(new HitWrapper(
hit.getString("_index"),
hit.getString("_type"),
hit.getString("_id"),
data.toString()));
final Object data = hit.opt("_source") != null ? hit.opt("_source") : hit.opt("fields");
response.addHit(
new HitWrapper(
hit.getString("_index"),
hit.getString("_type"),
hit.getString("_id"),
data.toString()));
}
}
} else {
@ -339,14 +342,13 @@ public class HttpBasedClient implements ElasticsearchClient {
}
private boolean containsAggs(HttpResponse<JsonNode> result) {
return result.getBody() != null &&
(result.getBody().getObject().has("aggregations") ||
result.getBody().getObject().has("aggs"));
return result.getBody() != null
&& (result.getBody().getObject().has("aggregations")
|| result.getBody().getObject().has("aggs"));
}
@Override
public void close() {
}
public void close() {}
@Override
public String toString() {

113
elasticsearch/src/main/java/org/apache/zeppelin/elasticsearch/client/TransportBasedClient.java
View file

@ -20,8 +20,21 @@ package org.apache.zeppelin.elasticsearch.client;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonSyntaxException;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter;
import org.apache.zeppelin.elasticsearch.action.ActionResponse;
import org.apache.zeppelin.elasticsearch.action.AggWrapper;
import org.apache.zeppelin.elasticsearch.action.HitWrapper;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
@ -45,96 +58,63 @@ import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregat
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter;
import org.apache.zeppelin.elasticsearch.action.ActionResponse;
import org.apache.zeppelin.elasticsearch.action.AggWrapper;
import org.apache.zeppelin.elasticsearch.action.HitWrapper;
/**
* Elasticsearch client using the transport protocol.
*/
/** Elasticsearch client using the transport protocol. */
public class TransportBasedClient implements ElasticsearchClient {
private final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private final Client client;
public TransportBasedClient(Properties props) throws UnknownHostException {
final String host =
props.getProperty(ElasticsearchInterpreter.ELASTICSEARCH_HOST);
final int port = Integer.parseInt(
props.getProperty(ElasticsearchInterpreter.ELASTICSEARCH_PORT));
final String host = props.getProperty(ElasticsearchInterpreter.ELASTICSEARCH_HOST);
final int port =
Integer.parseInt(props.getProperty(ElasticsearchInterpreter.ELASTICSEARCH_PORT));
final String clusterName =
props.getProperty(ElasticsearchInterpreter.ELASTICSEARCH_CLUSTER_NAME);
final Settings settings = Settings.settingsBuilder()
.put("cluster.name", clusterName)
.put(props)
.build();
final Settings settings =
Settings.settingsBuilder().put("cluster.name", clusterName).put(props).build();
client = TransportClient.builder().settings(settings).build()
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
client =
TransportClient.builder()
.settings(settings)
.build()
.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
}
@Override
public ActionResponse get(String index, String type, String id) {
final GetResponse getResp = client
.prepareGet(index, type, id)
.get();
final GetResponse getResp = client.prepareGet(index, type, id).get();
return new ActionResponse()
.succeeded(getResp.isExists())
.hit(new HitWrapper(
getResp.getIndex(),
getResp.getType(),
getResp.getId(),
getResp.getSourceAsString()));
.hit(
new HitWrapper(
getResp.getIndex(),
getResp.getType(),
getResp.getId(),
getResp.getSourceAsString()));
}
@Override
public ActionResponse delete(String index, String type, String id) {
final DeleteResponse delResp = client
.prepareDelete(index, type, id)
.get();
final DeleteResponse delResp = client.prepareDelete(index, type, id).get();
return new ActionResponse()
.succeeded(delResp.isFound())
.hit(new HitWrapper(
delResp.getIndex(),
delResp.getType(),
delResp.getId(),
null));
.hit(new HitWrapper(delResp.getIndex(), delResp.getType(), delResp.getId(), null));
}
@Override
public ActionResponse index(String index, String type, String id, String data) {
final IndexResponse idxResp = client
.prepareIndex(index, type, id)
.setSource(data)
.get();
final IndexResponse idxResp = client.prepareIndex(index, type, id).setSource(data).get();
return new ActionResponse()
.succeeded(idxResp.isCreated())
.hit(new HitWrapper(
idxResp.getIndex(),
idxResp.getType(),
idxResp.getId(),
null));
.hit(new HitWrapper(idxResp.getIndex(), idxResp.getType(), idxResp.getId(), null));
}
@Override
public ActionResponse search(String[] indices, String[] types, String query, int size) {
final SearchRequestBuilder reqBuilder = new SearchRequestBuilder(
client, SearchAction.INSTANCE);
final SearchRequestBuilder reqBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
reqBuilder.setIndices();
if (indices != null) {
@ -161,14 +141,13 @@ public class TransportBasedClient implements ElasticsearchClient {
final SearchResponse searchResp = reqBuilder.get();
final ActionResponse actionResp = new ActionResponse()
.succeeded(true)
.totalHits(searchResp.getHits().getTotalHits());
final ActionResponse actionResp =
new ActionResponse().succeeded(true).totalHits(searchResp.getHits().getTotalHits());
if (searchResp.getAggregations() != null) {
setAggregations(searchResp.getAggregations(), actionResp);
} else {
for (final SearchHit hit: searchResp.getHits()) {
for (final SearchHit hit : searchResp.getHits()) {
// Fields can be found either in _source, or in fields (it depends on the query)
// => specific for elasticsearch's version < 5
//
@ -193,11 +172,15 @@ public class TransportBasedClient implements ElasticsearchClient {
final Aggregation agg = aggregations.asList().get(0);
if (agg instanceof InternalMetricsAggregation) {
actionResp.addAggregation(new AggWrapper(AggWrapper.AggregationType.SIMPLE,
XContentHelper.toString((InternalMetricsAggregation) agg).toString()));
actionResp.addAggregation(
new AggWrapper(
AggWrapper.AggregationType.SIMPLE,
XContentHelper.toString((InternalMetricsAggregation) agg).toString()));
} else if (agg instanceof InternalSingleBucketAggregation) {
actionResp.addAggregation(new AggWrapper(AggWrapper.AggregationType.SIMPLE,
XContentHelper.toString((InternalSingleBucketAggregation) agg).toString()));
actionResp.addAggregation(
new AggWrapper(
AggWrapper.AggregationType.SIMPLE,
XContentHelper.toString((InternalSingleBucketAggregation) agg).toString()));
} else if (agg instanceof InternalMultiBucketAggregation) {
final Set<String> headerKeys = new HashSet<>();
final List<Map<String, Object>> buckets = new LinkedList<>();

210
elasticsearch/src/test/java/org/apache/zeppelin/elasticsearch/ElasticsearchInterpreterTest.java
View file

@ -21,7 +21,21 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang.math.RandomUtils;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
@ -35,22 +49,6 @@ import org.junit.experimental.theories.Theories;
import org.junit.experimental.theories.Theory;
import org.junit.runner.RunWith;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@RunWith(Theories.class)
public class ElasticsearchInterpreterTest {
@DataPoint public static ElasticsearchInterpreter transportInterpreter;
@ -59,8 +57,8 @@ public class ElasticsearchInterpreterTest {
private static Client elsClient;
private static Node elsNode;
private static final String[] METHODS = { "GET", "PUT", "DELETE", "POST" };
private static final int[] STATUS = { 200, 404, 500, 403 };
private static final String[] METHODS = {"GET", "PUT", "DELETE", "POST"};
private static final int[] STATUS = {200, 404, 500, 403};
private static final String ELS_CLUSTER_NAME = "zeppelin-elasticsearch-interpreter-test";
private static final String ELS_HOST = "localhost";
@ -72,7 +70,8 @@ public class ElasticsearchInterpreterTest {
@BeforeClass
public static void populate() throws IOException {
final Settings settings = Settings.settingsBuilder()
final Settings settings =
Settings.settingsBuilder()
.put("cluster.name", ELS_CLUSTER_NAME)
.put("network.host", ELS_HOST)
.put("http.port", ELS_HTTP_PORT)
@ -83,46 +82,58 @@ public class ElasticsearchInterpreterTest {
elsNode = NodeBuilder.nodeBuilder().settings(settings).node();
elsClient = elsNode.client();
elsClient.admin().indices().prepareCreate("logs")
.addMapping("http", jsonBuilder()
.startObject().startObject("http").startObject("properties")
.startObject("content_length")
.field("type", "integer")
.endObject()
.endObject().endObject().endObject()).get();
elsClient
.admin()
.indices()
.prepareCreate("logs")
.addMapping(
"http",
jsonBuilder()
.startObject()
.startObject("http")
.startObject("properties")
.startObject("content_length")
.field("type", "integer")
.endObject()
.endObject()
.endObject()
.endObject())
.get();
for (int i = 0; i < 48; i++) {
elsClient.prepareIndex("logs", "http", "" + i)
.setRefresh(true)
.setSource(jsonBuilder()
.startObject()
.field("date", new Date())
.startObject("request")
.field("method", METHODS[RandomUtils.nextInt(METHODS.length)])
.field("url", "/zeppelin/" + UUID.randomUUID().toString())
.field("headers", Arrays.asList("Accept: *.*", "Host: apache.org"))
.endObject()
.field("status", STATUS[RandomUtils.nextInt(STATUS.length)])
.field("content_length", RandomUtils.nextInt(2000))
)
.get();
elsClient
.prepareIndex("logs", "http", "" + i)
.setRefresh(true)
.setSource(
jsonBuilder()
.startObject()
.field("date", new Date())
.startObject("request")
.field("method", METHODS[RandomUtils.nextInt(METHODS.length)])
.field("url", "/zeppelin/" + UUID.randomUUID().toString())
.field("headers", Arrays.asList("Accept: *.*", "Host: apache.org"))
.endObject()
.field("status", STATUS[RandomUtils.nextInt(STATUS.length)])
.field("content_length", RandomUtils.nextInt(2000)))
.get();
}
for (int i = 1; i < 3; i++) {
elsClient.prepareIndex("logs", "http", "very/strange/id#" + i)
.setRefresh(true)
.setSource(jsonBuilder()
.startObject()
.field("date", new Date())
.startObject("request")
.field("method", METHODS[RandomUtils.nextInt(METHODS.length)])
.field("url", "/zeppelin/" + UUID.randomUUID().toString())
.field("headers", Arrays.asList("Accept: *.*", "Host: apache.org"))
.endObject()
.field("status", STATUS[RandomUtils.nextInt(STATUS.length)])
.field("content_length", RandomUtils.nextInt(2000))
)
.get();
elsClient
.prepareIndex("logs", "http", "very/strange/id#" + i)
.setRefresh(true)
.setSource(
jsonBuilder()
.startObject()
.field("date", new Date())
.startObject("request")
.field("method", METHODS[RandomUtils.nextInt(METHODS.length)])
.field("url", "/zeppelin/" + UUID.randomUUID().toString())
.field("headers", Arrays.asList("Accept: *.*", "Host: apache.org"))
.endObject()
.field("status", STATUS[RandomUtils.nextInt(STATUS.length)])
.field("content_length", RandomUtils.nextInt(2000)))
.get();
}
final Properties props = new Properties();
@ -181,8 +192,8 @@ public class ElasticsearchInterpreterTest {
assertNotNull(ctx.getAngularObjectRegistry().get("count_testCount", null, null));
assertEquals(50L, ctx.getAngularObjectRegistry().get("count_testCount", null, null).get());
res = interpreter.interpret("count /logs { \"query\": { \"match\": { \"status\": 500 } } }",
ctx);
res =
interpreter.interpret("count /logs { \"query\": { \"match\": { \"status\": 500 } } }", ctx);
assertEquals(Code.SUCCESS, res.code());
}
@ -219,15 +230,19 @@ public class ElasticsearchInterpreterTest {
res = interpreter.interpret("search /logs {{{hello}}}", ctx);
assertEquals(Code.ERROR, res.code());
res = interpreter.interpret("search /logs { \"query\": { \"match\": { \"status\": 500 } } }",
ctx);
res =
interpreter.interpret(
"search /logs { \"query\": { \"match\": { \"status\": 500 } } }", ctx);
assertEquals(Code.SUCCESS, res.code());
res = interpreter.interpret("search /logs status:404", ctx);
assertEquals(Code.SUCCESS, res.code());
res = interpreter.interpret("search /logs { \"fields\": [ \"date\", \"request.headers\" ], " +
"\"query\": { \"match\": { \"status\": 500 } } }", ctx);
res =
interpreter.interpret(
"search /logs { \"fields\": [ \"date\", \"request.headers\" ], "
+ "\"query\": { \"match\": { \"status\": 500 } } }",
ctx);
assertEquals(Code.SUCCESS, res.code());
}
@ -236,50 +251,75 @@ public class ElasticsearchInterpreterTest {
final InterpreterContext ctx = buildContext("agg");
// Single-value metric
InterpreterResult res = interpreter.interpret("search /logs { \"aggs\" : " +
"{ \"distinct_status_count\" : " +
" { \"cardinality\" : { \"field\" : \"status\" } } } }", ctx);
InterpreterResult res =
interpreter.interpret(
"search /logs { \"aggs\" : "
+ "{ \"distinct_status_count\" : "
+ " { \"cardinality\" : { \"field\" : \"status\" } } } }",
ctx);
assertEquals(Code.SUCCESS, res.code());
// Multi-value metric
res = interpreter.interpret("search /logs { \"aggs\" : { \"content_length_stats\" : " +
" { \"extended_stats\" : { \"field\" : \"content_length\" } } } }", ctx);
res =
interpreter.interpret(
"search /logs { \"aggs\" : { \"content_length_stats\" : "
+ " { \"extended_stats\" : { \"field\" : \"content_length\" } } } }",
ctx);
assertEquals(Code.SUCCESS, res.code());
// Single bucket
res = interpreter.interpret("search /logs { \"aggs\" : { " +
" \"200_OK\" : { \"filter\" : { \"term\": { \"status\": \"200\" } }, " +
" \"aggs\" : { \"avg_length\" : { \"avg\" : " +
"{ \"field\" : \"content_length\" } } } } } }", ctx);
res =
interpreter.interpret(
"search /logs { \"aggs\" : { "
+ " \"200_OK\" : { \"filter\" : { \"term\": { \"status\": \"200\" } }, "
+ " \"aggs\" : { \"avg_length\" : { \"avg\" : "
+ "{ \"field\" : \"content_length\" } } } } } }",
ctx);
assertEquals(Code.SUCCESS, res.code());
// Multi-buckets
res = interpreter.interpret("search /logs { \"aggs\" : { \"status_count\" : " +
" { \"terms\" : { \"field\" : \"status\" } } } }", ctx);
res =
interpreter.interpret(
"search /logs { \"aggs\" : { \"status_count\" : "
+ " { \"terms\" : { \"field\" : \"status\" } } } }",
ctx);
assertEquals(Code.SUCCESS, res.code());
res = interpreter.interpret("search /logs { \"aggs\" : { " +
" \"length\" : { \"terms\": { \"field\": \"status\" }, " +
" \"aggs\" : { \"sum_length\" : { \"sum\" : { \"field\" : \"content_length\" } }, " +
"\"sum_status\" : { \"sum\" : { \"field\" : \"status\" } } } } } }", ctx);
res =
interpreter.interpret(
"search /logs { \"aggs\" : { "
+ " \"length\" : { \"terms\": { \"field\": \"status\" }, "
+ " \"aggs\" : { \"sum_length\" : { \"sum\" : { \"field\" : \"content_length\" } }, "
+ "\"sum_status\" : { \"sum\" : { \"field\" : \"status\" } } } } } }",
ctx);
assertEquals(Code.SUCCESS, res.code());
}
@Theory
public void testIndex(ElasticsearchInterpreter interpreter) {
InterpreterResult res = interpreter.interpret("index /logs { \"date\": \"" + new Date() +
"\", \"method\": \"PUT\", \"status\": \"500\" }", null);
InterpreterResult res =
interpreter.interpret(
"index /logs { \"date\": \""
+ new Date()
+ "\", \"method\": \"PUT\", \"status\": \"500\" }",
null);
assertEquals(Code.ERROR, res.code());
res = interpreter.interpret("index /logs/http { bad ", null);
assertEquals(Code.ERROR, res.code());
res = interpreter.interpret("index /logs/http { \"date\": \"2015-12-06T14:54:23.368Z\", " +
"\"method\": \"PUT\", \"status\": \"500\" }", null);
res =
interpreter.interpret(
"index /logs/http { \"date\": \"2015-12-06T14:54:23.368Z\", "
+ "\"method\": \"PUT\", \"status\": \"500\" }",
null);
assertEquals(Code.SUCCESS, res.code());
res = interpreter.interpret("index /logs/http/1000 { \"date\": " +
"\"2015-12-06T14:54:23.368Z\", \"method\": \"PUT\", \"status\": \"500\" }", null);
res =
interpreter.interpret(
"index /logs/http/1000 { \"date\": "
+ "\"2015-12-06T14:54:23.368Z\", \"method\": \"PUT\", \"status\": \"500\" }",
null);
assertEquals(Code.SUCCESS, res.code());
}
@ -308,10 +348,10 @@ public class ElasticsearchInterpreterTest {
@Theory
public void testCompletion(ElasticsearchInterpreter interpreter) {
final List<InterpreterCompletion> expectedResultOne = Arrays.asList(
new InterpreterCompletion("count", "count", CompletionType.command.name()));
final List<InterpreterCompletion> expectedResultTwo = Arrays.asList(
new InterpreterCompletion("help", "help", CompletionType.command.name()));
final List<InterpreterCompletion> expectedResultOne =
Arrays.asList(new InterpreterCompletion("count", "count", CompletionType.command.name()));
final List<InterpreterCompletion> expectedResultTwo =
Arrays.asList(new InterpreterCompletion("help", "help", CompletionType.command.name()));
final List<InterpreterCompletion> resultOne = interpreter.completion("co", 0, null);
final List<InterpreterCompletion> resultTwo = interpreter.completion("he", 0, null);

7
file/pom.xml
View file

@ -91,13 +91,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

54
file/src/main/java/org/apache/zeppelin/file/FileInterpreter.java
View file

@ -1,26 +1,19 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.file;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
@ -28,7 +21,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Properties;
import java.util.StringTokenizer;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
@ -38,11 +30,10 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* File interpreter for Zeppelin.
*
*/
/** File interpreter for Zeppelin. */
public abstract class FileInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(FileInterpreter.class);
String currentDir = null;
@ -53,9 +44,7 @@ public abstract class FileInterpreter extends Interpreter {
currentDir = new String("/");
}
/**
* Handling the arguments of the command.
*/
/** Handling the arguments of the command. */
public class CommandArgs {
public String input = null;
public String command = null;
@ -69,12 +58,12 @@ public abstract class FileInterpreter extends Interpreter {
}
private void parseArg(String arg) {
if (arg.charAt(0) == '-') { // handle flags
if (arg.charAt(0) == '-') { // handle flags
for (int i = 0; i < arg.length(); i++) {
Character c = arg.charAt(i);
flags.add(c);
}
} else { // handle other args
} else { // handle other args
args.add(arg);
}
}
@ -99,7 +88,7 @@ public abstract class FileInterpreter extends Interpreter {
public abstract boolean isDirectory(String path);
// Combine paths, takes care of arguments such as ..
protected String getNewPath(String argument){
protected String getNewPath(String argument) {
Path arg = Paths.get(argument);
Path ret = arg.isAbsolute() ? arg : Paths.get(currentDir, argument);
return ret.normalize().toString();
@ -145,8 +134,7 @@ public abstract class FileInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {
}
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
@ -160,13 +148,13 @@ public abstract class FileInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
FileInterpreter.class.getName() + this.hashCode());
return SchedulerFactory.singleton()
.createOrGetFIFOScheduler(FileInterpreter.class.getName() + this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return null;
}
}

58
file/src/main/java/org/apache/zeppelin/file/HDFSCommand.java
View file

@ -1,47 +1,35 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.file;
import org.slf4j.Logger;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import javax.ws.rs.core.UriBuilder;
import org.slf4j.Logger;
/**
* Definition and HTTP invocation methods for all WebHDFS commands.
*/
/** Definition and HTTP invocation methods for all WebHDFS commands. */
public class HDFSCommand {
/**
* Type of HTTP request.
*/
/** Type of HTTP request. */
public enum HttpType {
GET,
PUT
}
/**
* Definition of WebHDFS operator.
*/
/** Definition of WebHDFS operator. */
public class Op {
public String op;
public HttpType cmd;
@ -54,9 +42,7 @@ public class HDFSCommand {
}
}
/**
* Definition of argument to an operator.
*/
/** Definition of argument to an operator. */
public class Arg {
public String key;
public String value;
@ -86,11 +72,9 @@ public class HDFSCommand {
}
public String checkArgs(Op op, String path, Arg[] args) throws Exception {
if (op == null ||
path == null ||
(op.minArgs > 0 &&
(args == null ||
args.length != op.minArgs))) {
if (op == null
|| path == null
|| (op.minArgs > 0 && (args == null || args.length != op.minArgs))) {
String a = "";
a = (op != null) ? a + op.op + "\n" : a;
a = (path != null) ? a + path + "\n" : a;
@ -110,10 +94,7 @@ public class HDFSCommand {
}
// Build URI
UriBuilder builder = UriBuilder
.fromPath(url)
.path(path)
.queryParam("op", op.op);
UriBuilder builder = UriBuilder.fromPath(url).path(path).queryParam("op", op.op);
if (args != null) {
for (Arg a : args) {
@ -132,8 +113,7 @@ public class HDFSCommand {
logger.info("Sending 'GET' request to URL : " + hdfsUrl);
logger.info("Response Code : " + responseCode);
StringBuffer response = new StringBuffer();
try (BufferedReader in = new BufferedReader(
new InputStreamReader(con.getInputStream()));) {
try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream())); ) {
String inputLine;
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);

118
file/src/main/java/org/apache/zeppelin/file/HDFSFileInterpreter.java
View file

@ -1,42 +1,33 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.file;
import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang.StringUtils;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
/**
* HDFS implementation of File interpreter for Zeppelin.
*/
/** HDFS implementation of File interpreter for Zeppelin. */
public class HDFSFileInterpreter extends FileInterpreter {
static final String HDFS_URL = "hdfs.url";
static final String HDFS_USER = "hdfs.user";
@ -54,7 +45,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
gson = new Gson();
}
public HDFSFileInterpreter(Properties property){
public HDFSFileInterpreter(Properties property) {
super(property);
prepare();
}
@ -62,7 +53,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
/**
* Status of one file.
*
* matches returned JSON
* <p>matches returned JSON
*/
public class OneFileStatus {
public long accessTime;
@ -101,7 +92,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
/**
* Status of one file.
*
* matches returned JSON
* <p>matches returned JSON
*/
public class SingleFileStatus {
@SerializedName("FileStatus")
@ -111,7 +102,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
/**
* Status of all files in a directory.
*
* matches returned JSON
* <p>matches returned JSON
*/
public class MultiFileStatus {
@SerializedName("FileStatus")
@ -121,7 +112,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
/**
* Status of all files in a directory.
*
* matches returned JSON
* <p>matches returned JSON
*/
public class AllFileStatus {
@SerializedName("FileStatuses")
@ -146,26 +137,25 @@ public class HDFSFileInterpreter extends FileInterpreter {
}
@Override
public void close() {
}
public void close() {}
private String listDir(String path) throws Exception {
return cmd.runCommand(cmd.listStatus, path, null);
}
private String listPermission(OneFileStatus fs){
private String listPermission(OneFileStatus fs) {
StringBuilder sb = new StringBuilder();
sb.append(fs.type.equalsIgnoreCase("Directory") ? 'd' : '-');
int p = Integer.parseInt(fs.permission, 16);
sb.append(((p & 0x400) == 0) ? '-' : 'r');
sb.append(((p & 0x200) == 0) ? '-' : 'w');
sb.append(((p & 0x100) == 0) ? '-' : 'x');
sb.append(((p & 0x40) == 0) ? '-' : 'r');
sb.append(((p & 0x20) == 0) ? '-' : 'w');
sb.append(((p & 0x10) == 0) ? '-' : 'x');
sb.append(((p & 0x4) == 0) ? '-' : 'r');
sb.append(((p & 0x2) == 0) ? '-' : 'w');
sb.append(((p & 0x1) == 0) ? '-' : 'x');
sb.append(((p & 0x40) == 0) ? '-' : 'r');
sb.append(((p & 0x20) == 0) ? '-' : 'w');
sb.append(((p & 0x10) == 0) ? '-' : 'x');
sb.append(((p & 0x4) == 0) ? '-' : 'r');
sb.append(((p & 0x2) == 0) ? '-' : 'w');
sb.append(((p & 0x1) == 0) ? '-' : 'x');
return sb.toString();
}
@ -180,7 +170,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
sb.append(((fs.replication == 0) ? "-" : fs.replication) + "\t ");
sb.append(fs.owner + "\t");
sb.append(fs.group + "\t");
if (args.flags.contains(new Character('h'))){ //human readable
if (args.flags.contains(new Character('h'))) { // human readable
sb.append(humanReadableByteCount(fs.length) + "\t\t");
} else {
sb.append(fs.length + "\t");
@ -224,17 +214,19 @@ public class HDFSFileInterpreter extends FileInterpreter {
}
try {
//see if directory.
// see if directory.
if (isDirectory(path)) {
String sfs = listDir(path);
if (sfs != null) {
AllFileStatus allFiles = gson.fromJson(sfs, AllFileStatus.class);
if (allFiles != null &&
allFiles.fileStatuses != null &&
allFiles.fileStatuses.fileStatus != null) {
int length = cmd.maxLength < allFiles.fileStatuses.fileStatus.length ? cmd.maxLength :
allFiles.fileStatuses.fileStatus.length;
if (allFiles != null
&& allFiles.fileStatuses != null
&& allFiles.fileStatuses.fileStatus != null) {
int length =
cmd.maxLength < allFiles.fileStatuses.fileStatus.length
? cmd.maxLength
: allFiles.fileStatuses.fileStatus.length;
for (int index = 0; index < length; index++) {
OneFileStatus fs = allFiles.fileStatuses.fileStatus[index];
all = all + listOne(path, fs) + '\n';
@ -271,8 +263,8 @@ public class HDFSFileInterpreter extends FileInterpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
logger.info("Completion request at position\t" + cursor + " in string " + buf);
final List<InterpreterCompletion> suggestions = new ArrayList<>();
if (StringUtils.isEmpty(buf)) {
@ -282,19 +274,16 @@ public class HDFSFileInterpreter extends FileInterpreter {
return suggestions;
}
//part of a command == no spaces
if (buf.split(" ").length == 1){
// part of a command == no spaces
if (buf.split(" ").length == 1) {
if ("cd".contains(buf)) {
suggestions.add(new InterpreterCompletion("cd", "cd",
CompletionType.command.name()));
suggestions.add(new InterpreterCompletion("cd", "cd", CompletionType.command.name()));
}
if ("ls".contains(buf)) {
suggestions.add(new InterpreterCompletion("ls", "ls",
CompletionType.command.name()));
suggestions.add(new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
}
if ("pwd".contains(buf)) {
suggestions.add(new InterpreterCompletion("pwd", "pwd",
CompletionType.command.name()));
suggestions.add(new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
}
return suggestions;
@ -302,35 +291,36 @@ public class HDFSFileInterpreter extends FileInterpreter {
// last word will contain the path we're working with.
String lastToken = buf.substring(buf.lastIndexOf(" ") + 1);
if (lastToken.startsWith("-")) { //flag not path
if (lastToken.startsWith("-")) { // flag not path
return null;
}
String localPath = ""; //all things before the last '/'
String unfinished = lastToken; //unfished filenames or directories
String localPath = ""; // all things before the last '/'
String unfinished = lastToken; // unfished filenames or directories
if (lastToken.contains("/")) {
localPath = lastToken.substring(0, lastToken.lastIndexOf('/') + 1);
unfinished = lastToken.substring(lastToken.lastIndexOf('/') + 1);
}
String globalPath = getNewPath(localPath); //adjust for cwd
String globalPath = getNewPath(localPath); // adjust for cwd
if (isDirectory(globalPath)){
if (isDirectory(globalPath)) {
try {
String fileStatusString = listDir(globalPath);
if (fileStatusString != null) {
AllFileStatus allFiles = gson.fromJson(fileStatusString, AllFileStatus.class);
if (allFiles != null &&
allFiles.fileStatuses != null &&
allFiles.fileStatuses.fileStatus != null) {
if (allFiles != null
&& allFiles.fileStatuses != null
&& allFiles.fileStatuses.fileStatus != null) {
for (OneFileStatus fs : allFiles.fileStatuses.fileStatus) {
if (fs.pathSuffix.contains(unfinished)) {
//only suggest the text after the last .
// only suggest the text after the last .
String beforeLastPeriod = unfinished.substring(0, unfinished.lastIndexOf('.') + 1);
//beforeLastPeriod should be the start of fs.pathSuffix, so take the end of it.
// beforeLastPeriod should be the start of fs.pathSuffix, so take the end of it.
String suggestedFinish = fs.pathSuffix.substring(beforeLastPeriod.length());
suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish,
CompletionType.path.name()));
suggestions.add(
new InterpreterCompletion(
suggestedFinish, suggestedFinish, CompletionType.path.name()));
}
}
return suggestions;
@ -344,7 +334,7 @@ public class HDFSFileInterpreter extends FileInterpreter {
logger.info("path is not a directory. No values suggested.");
}
//Error in string.
// Error in string.
return null;
}
}

215
file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java
View file

@ -1,44 +1,34 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.file;
import static org.junit.Assert.assertNull;
import com.google.gson.Gson;
import junit.framework.TestCase;
import org.junit.Test;
import org.slf4j.Logger;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
import junit.framework.TestCase;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.junit.Test;
import org.slf4j.Logger;
/**
* Tests Interpreter by running pre-determined commands against mock file system.
*/
/** Tests Interpreter by running pre-determined commands against mock file system. */
public class HDFSFileInterpreterTest extends TestCase {
@Test
public void testMaxLength() {
@ -121,10 +111,10 @@ public class HDFSFileInterpreterTest extends TestCase {
assertEquals(result1.message().get(0).getData(), result11.message().get(0).getData());
// auto completion test
List expectedResultOne = Arrays.asList(
new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
List expectedResultTwo = Arrays.asList(
new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
List expectedResultOne =
Arrays.asList(new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
List expectedResultTwo =
Arrays.asList(new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
List<InterpreterCompletion> resultOne = t.completion("l", 0, null);
List<InterpreterCompletion> resultTwo = t.completion("p", 0, null);
@ -135,93 +125,93 @@ public class HDFSFileInterpreterTest extends TestCase {
}
}
/**
* Store command results from curl against a real file system.
*/
/** Store command results from curl against a real file system. */
class MockFileSystem {
HashMap<String, String> mfs = new HashMap<>();
static final String FILE_STATUSES =
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16389," +
"\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1438548219672," +
"\"owner\":\"yarn\",\"pathSuffix\":\"app-logs\",\"permission\":\"777\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" +
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16395," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548030045," +
"\"owner\":\"hdfs\",\"pathSuffix\":\"hdp\",\"permission\":\"755\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" +
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16390," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547985336," +
"\"owner\":\"mapred\",\"pathSuffix\":\"mapred\",\"permission\":\"755\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" +
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":2,\"fileId\":16392," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547985346," +
"\"owner\":\"hdfs\",\"pathSuffix\":\"mr-history\",\"permission\":\"755\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" +
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16400," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548089725," +
"\"owner\":\"hdfs\",\"pathSuffix\":\"system\",\"permission\":\"755\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" +
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16386," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548150089," +
"\"owner\":\"hdfs\",\"pathSuffix\":\"tmp\",\"permission\":\"777\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n" +
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16387," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547921792," +
"\"owner\":\"hdfs\",\"pathSuffix\":\"user\",\"permission\":\"755\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n";
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16389,"
+ "\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1438548219672,"
+ "\"owner\":\"yarn\",\"pathSuffix\":\"app-logs\",\"permission\":\"777\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n"
+ "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16395,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548030045,"
+ "\"owner\":\"hdfs\",\"pathSuffix\":\"hdp\",\"permission\":\"755\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n"
+ "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16390,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547985336,"
+ "\"owner\":\"mapred\",\"pathSuffix\":\"mapred\",\"permission\":\"755\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n"
+ "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":2,\"fileId\":16392,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547985346,"
+ "\"owner\":\"hdfs\",\"pathSuffix\":\"mr-history\",\"permission\":\"755\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n"
+ "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16400,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548089725,"
+ "\"owner\":\"hdfs\",\"pathSuffix\":\"system\",\"permission\":\"755\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n"
+ "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16386,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548150089,"
+ "\"owner\":\"hdfs\",\"pathSuffix\":\"tmp\",\"permission\":\"777\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"},\n"
+ "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16387,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438547921792,"
+ "\"owner\":\"hdfs\",\"pathSuffix\":\"user\",\"permission\":\"755\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n";
void addListStatusData() {
mfs.put("/?op=LISTSTATUS",
"{\"FileStatuses\":{\"FileStatus\":[\n" + FILE_STATUSES +
"]}}"
);
mfs.put("/user?op=LISTSTATUS", "{\"FileStatuses\":{\"FileStatus\":[\n" +
" {\"accessTime\":0,\"blockSize\":0,\"childrenNum\":4,\"fileId\":16388," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253161263," +
"\"owner\":\"ambari-qa\",\"pathSuffix\":\"ambari-qa\",\"permission\":\"770\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n" +
" ]}}"
);
mfs.put("/tmp?op=LISTSTATUS",
"{\"FileStatuses\":{\"FileStatus\":[\n" +
" {\"accessTime\":1441253097489,\"blockSize\":134217728,\"childrenNum\":0," +
"\"fileId\":16400,\"group\":\"hdfs\",\"length\":1645," +
"\"modificationTime\":1441253097517,\"owner\":\"hdfs\"," +
"\"pathSuffix\":\"ida8c06540_date040315\",\"permission\":\"755\"," +
"\"replication\":3,\"storagePolicy\":0,\"type\":\"FILE\"}\n" +
" ]}}"
);
mfs.put("/mr-history/done?op=LISTSTATUS",
"{\"FileStatuses\":{\"FileStatus\":[\n" +
"{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16433," +
"\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197481," +
"\"owner\":\"mapred\",\"pathSuffix\":\"2015\",\"permission\":\"770\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n" +
"]}}"
);
mfs.put("/?op=LISTSTATUS", "{\"FileStatuses\":{\"FileStatus\":[\n" + FILE_STATUSES + "]}}");
mfs.put(
"/user?op=LISTSTATUS",
"{\"FileStatuses\":{\"FileStatus\":[\n"
+ " {\"accessTime\":0,\"blockSize\":0,\"childrenNum\":4,\"fileId\":16388,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253161263,"
+ "\"owner\":\"ambari-qa\",\"pathSuffix\":\"ambari-qa\",\"permission\":\"770\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n"
+ " ]}}");
mfs.put(
"/tmp?op=LISTSTATUS",
"{\"FileStatuses\":{\"FileStatus\":[\n"
+ " {\"accessTime\":1441253097489,\"blockSize\":134217728,\"childrenNum\":0,"
+ "\"fileId\":16400,\"group\":\"hdfs\",\"length\":1645,"
+ "\"modificationTime\":1441253097517,\"owner\":\"hdfs\","
+ "\"pathSuffix\":\"ida8c06540_date040315\",\"permission\":\"755\","
+ "\"replication\":3,\"storagePolicy\":0,\"type\":\"FILE\"}\n"
+ " ]}}");
mfs.put(
"/mr-history/done?op=LISTSTATUS",
"{\"FileStatuses\":{\"FileStatus\":[\n"
+ "{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16433,"
+ "\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197481,"
+ "\"owner\":\"mapred\",\"pathSuffix\":\"2015\",\"permission\":\"770\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}\n"
+ "]}}");
}
void addGetFileStatusData() {
mfs.put("/?op=GETFILESTATUS",
"{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":7,\"fileId\":16385," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548089725," +
"\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"755\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
mfs.put("/user?op=GETFILESTATUS",
"{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16387," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253043188," +
"\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"755\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
mfs.put("/tmp?op=GETFILESTATUS",
"{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16386," +
"\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253097489," +
"\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"777\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
mfs.put("/mr-history/done?op=GETFILESTATUS",
"{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16393," +
"\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197480," +
"\"owner\":\"mapred\",\"pathSuffix\":\"\",\"permission\":\"777\"," +
"\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
mfs.put(
"/?op=GETFILESTATUS",
"{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":7,\"fileId\":16385,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1438548089725,"
+ "\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"755\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
mfs.put(
"/user?op=GETFILESTATUS",
"{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16387,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253043188,"
+ "\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"755\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
mfs.put(
"/tmp?op=GETFILESTATUS",
"{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16386,"
+ "\"group\":\"hdfs\",\"length\":0,\"modificationTime\":1441253097489,"
+ "\"owner\":\"hdfs\",\"pathSuffix\":\"\",\"permission\":\"777\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
mfs.put(
"/mr-history/done?op=GETFILESTATUS",
"{\"FileStatus\":{\"accessTime\":0,\"blockSize\":0,\"childrenNum\":1,\"fileId\":16393,"
+ "\"group\":\"hadoop\",\"length\":0,\"modificationTime\":1441253197480,"
+ "\"owner\":\"mapred\",\"pathSuffix\":\"\",\"permission\":\"777\","
+ "\"replication\":0,\"storagePolicy\":0,\"type\":\"DIRECTORY\"}}");
}
public void addMockData(HDFSCommand.Op op) {
@ -238,9 +228,7 @@ class MockFileSystem {
}
}
/**
* Run commands against mock file system that simulates webhdfs responses.
*/
/** Run commands against mock file system that simulates webhdfs responses. */
class MockHDFSCommand extends HDFSCommand {
MockFileSystem fs = null;
@ -271,15 +259,14 @@ class MockHDFSCommand extends HDFSCommand {
}
}
/**
* Mock Interpreter - uses Mock HDFS command.
*/
/** Mock Interpreter - uses Mock HDFS command. */
class MockHDFSFileInterpreter extends HDFSFileInterpreter {
@Override
public void prepare() {
// Run commands against mock File System instead of WebHDFS
int i = Integer.parseInt(getProperty(HDFS_MAXLENGTH) == null ? "1000"
: getProperty(HDFS_MAXLENGTH));
int i =
Integer.parseInt(
getProperty(HDFS_MAXLENGTH) == null ? "1000" : getProperty(HDFS_MAXLENGTH));
cmd = new MockHDFSCommand("", "", logger, i);
gson = new Gson();
}

8
flink/pom.xml
View file

@ -297,14 +297,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

25
flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java
View file

@ -17,6 +17,9 @@
package org.apache.zeppelin.flink;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.flink.api.scala.ExecutionEnvironment;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
@ -24,10 +27,6 @@ import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
public class FlinkInterpreter extends Interpreter {
private FlinkScalaInterpreter innerIntp;
@ -44,8 +43,11 @@ public class FlinkInterpreter extends Interpreter {
// bind ZeppelinContext
int maxRow = Integer.parseInt(getProperty("zeppelin.flink.maxResult", "1000"));
this.z = new FlinkZeppelinContext(innerIntp.getBatchTableEnviroment(),
getInterpreterGroup().getInterpreterHookRegistry(), maxRow);
this.z =
new FlinkZeppelinContext(
innerIntp.getBatchTableEnviroment(),
getInterpreterGroup().getInterpreterHookRegistry(),
maxRow);
List<String> modifiers = new ArrayList<>();
modifiers.add("@transient");
this.innerIntp.bind("z", z.getClass().getCanonicalName(), z, modifiers);
@ -66,9 +68,7 @@ public class FlinkInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) throws InterpreterException {
}
public void cancel(InterpreterContext context) throws InterpreterException {}
@Override
public FormType getFormType() throws InterpreterException {
@ -81,10 +81,8 @@ public class FlinkInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf,
int cursor,
InterpreterContext interpreterContext)
throws InterpreterException {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) throws InterpreterException {
return innerIntp.completion(buf, cursor, interpreterContext);
}
@ -99,5 +97,4 @@ public class FlinkInterpreter extends Interpreter {
FlinkZeppelinContext getZeppelinContext() {
return this.z;
}
}

17
flink/src/main/java/org/apache/zeppelin/flink/FlinkSQLInterpreter.java
View file

@ -17,14 +17,12 @@
package org.apache.zeppelin.flink;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import java.util.Properties;
public class FlinkSQLInterpreter extends Interpreter {
private FlinkSQLScalaInterpreter sqlScalaInterpreter;
@ -33,21 +31,18 @@ public class FlinkSQLInterpreter extends Interpreter {
super(properties);
}
@Override
public void open() throws InterpreterException {
FlinkInterpreter flinkInterpreter =
getInterpreterInTheSameSessionByClassName(FlinkInterpreter.class);
FlinkZeppelinContext z = flinkInterpreter.getZeppelinContext();
int maxRow = Integer.parseInt(getProperty("zeppelin.flink.maxResult", "1000"));
this.sqlScalaInterpreter = new FlinkSQLScalaInterpreter(
flinkInterpreter.getInnerScalaInterpreter(), z, maxRow);
this.sqlScalaInterpreter =
new FlinkSQLScalaInterpreter(flinkInterpreter.getInnerScalaInterpreter(), z, maxRow);
}
@Override
public void close() throws InterpreterException {
}
public void close() throws InterpreterException {}
@Override
public InterpreterResult interpret(String st, InterpreterContext context)
@ -56,9 +51,7 @@ public class FlinkSQLInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) throws InterpreterException {
}
public void cancel(InterpreterContext context) throws InterpreterException {}
@Override
public FormType getFormType() throws InterpreterException {

143
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java
View file

@ -16,7 +16,15 @@
*/
package org.apache.zeppelin.flink;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.ui.CheckBox;
import org.apache.zeppelin.display.ui.Select;
@ -33,16 +41,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class FlinkInterpreterTest {
private FlinkInterpreter interpreter;
@ -71,8 +69,8 @@ public class FlinkInterpreterTest {
@Test
public void testBasicScala() throws InterpreterException, IOException {
InterpreterResult result = interpreter.interpret("val a=\"hello world\"",
getInterpreterContext());
InterpreterResult result =
interpreter.interpret("val a=\"hello world\"", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals("a: String = hello world\n", output);
@ -102,38 +100,42 @@ public class FlinkInterpreterTest {
result = interpreter.interpret("/*comment here*/", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
result = interpreter.interpret("/*comment here*/\nprint(\"hello world\")",
getInterpreterContext());
result =
interpreter.interpret("/*comment here*/\nprint(\"hello world\")", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
// multiple line comment
result = interpreter.interpret("/*line 1 \n line 2*/",
getInterpreterContext());
result = interpreter.interpret("/*line 1 \n line 2*/", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
// test function
result = interpreter.interpret("def add(x:Int, y:Int)\n{ return x+y }",
getInterpreterContext());
result =
interpreter.interpret("def add(x:Int, y:Int)\n{ return x+y }", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
result = interpreter.interpret("print(add(1,2))", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
result = interpreter.interpret("/*line 1 \n line 2*/print(\"hello world\")",
getInterpreterContext());
result =
interpreter.interpret(
"/*line 1 \n line 2*/print(\"hello world\")", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
// companion object
result = interpreter.interpret("class Counter {\n " +
"var value: Long = 0} \n" +
"object Counter {\n def apply(x: Long) = new Counter()\n}", getInterpreterContext());
result =
interpreter.interpret(
"class Counter {\n "
+ "var value: Long = 0} \n"
+ "object Counter {\n def apply(x: Long) = new Counter()\n}",
getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
// case class
result = interpreter.interpret(
"case class Bank(age:Integer, job:String, marital : String, education : String," +
" balance : Integer)\n",
getInterpreterContext());
result =
interpreter.interpret(
"case class Bank(age:Integer, job:String, marital : String, education : String,"
+ " balance : Integer)\n",
getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
// ZeppelinContext
@ -141,14 +143,12 @@ public class FlinkInterpreterTest {
result = interpreter.interpret("val ds = benv.fromElements(1,2,3)\nz.show(ds)", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, messageOutput.get(0).getType());
assertEquals("f0\n" +
"1\n" +
"2\n" +
"3\n", messageOutput.get(0).toInterpreterResultMessage().getData());
assertEquals(
"f0\n" + "1\n" + "2\n" + "3\n",
messageOutput.get(0).toInterpreterResultMessage().getData());
context = getInterpreterContext();
result = interpreter.interpret("z.input(\"name\", \"default_name\")",
context);
result = interpreter.interpret("z.input(\"name\", \"default_name\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, context.getGui().getForms().size());
assertTrue(context.getGui().getForms().get("name") instanceof TextBox);
@ -157,8 +157,11 @@ public class FlinkInterpreterTest {
assertEquals("default_name", textBox.getDefaultValue());
context = getInterpreterContext();
result = interpreter.interpret("z.checkbox(\"checkbox_1\", " +
"Seq(\"value_2\"), Seq((\"value_1\", \"name_1\"), (\"value_2\", \"name_2\")))", context);
result =
interpreter.interpret(
"z.checkbox(\"checkbox_1\", "
+ "Seq(\"value_2\"), Seq((\"value_1\", \"name_1\"), (\"value_2\", \"name_2\")))",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, context.getGui().getForms().size());
assertTrue(context.getGui().getForms().get("checkbox_1") instanceof CheckBox);
@ -173,8 +176,11 @@ public class FlinkInterpreterTest {
assertEquals("name_2", checkBox.getOptions()[1].getDisplayName());
context = getInterpreterContext();
result = interpreter.interpret("z.select(\"select_1\", Seq(\"value_2\"), " +
"Seq((\"value_1\", \"name_1\"), (\"value_2\", \"name_2\")))", context);
result =
interpreter.interpret(
"z.select(\"select_1\", Seq(\"value_2\"), "
+ "Seq((\"value_1\", \"name_1\"), (\"value_2\", \"name_2\")))",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, context.getGui().getForms().size());
assertTrue(context.getGui().getForms().get("select_1") instanceof Select);
@ -192,24 +198,25 @@ public class FlinkInterpreterTest {
@Test
public void testCompletion() throws InterpreterException {
InterpreterResult result = interpreter.interpret("val a=\"hello world\"",
getInterpreterContext());
InterpreterResult result =
interpreter.interpret("val a=\"hello world\"", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals("a: String = hello world\n", output);
List<InterpreterCompletion> completions = interpreter.completion("a.", 2,
getInterpreterContext());
List<InterpreterCompletion> completions =
interpreter.completion("a.", 2, getInterpreterContext());
assertTrue(completions.size() > 0);
}
// Disable it for now as there's extra std output from flink shell.
@Test
public void testWordCount() throws InterpreterException, IOException {
interpreter.interpret("val text = benv.fromElements(\"To be or not to be\")",
interpreter.interpret(
"val text = benv.fromElements(\"To be or not to be\")", getInterpreterContext());
interpreter.interpret(
"val counts = text.flatMap { _.toLowerCase.split(\" \") }"
+ ".map { (_, 1) }.groupBy(0).sum(1)",
getInterpreterContext());
interpreter.interpret("val counts = text.flatMap { _.toLowerCase.split(\" \") }" +
".map { (_, 1) }.groupBy(0).sum(1)", getInterpreterContext());
InterpreterResult result = interpreter.interpret("counts.print()", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@ -225,31 +232,31 @@ public class FlinkInterpreterTest {
private InterpreterContext getInterpreterContext() {
output = "";
messageOutput = new ArrayList<>();
InterpreterContext context = InterpreterContext.builder()
.setInterpreterOut(new InterpreterOutput(null))
.setAngularObjectRegistry(new AngularObjectRegistry("flink", null))
.build();
context.out = new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onUpdateAll(InterpreterOutput out) {
InterpreterContext context =
InterpreterContext.builder()
.setInterpreterOut(new InterpreterOutput(null))
.setAngularObjectRegistry(new AngularObjectRegistry("flink", null))
.build();
context.out =
new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onUpdateAll(InterpreterOutput out) {}
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
try {
output = out.toInterpreterResultMessage().getData();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
try {
output = out.toInterpreterResultMessage().getData();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
messageOutput.add(out);
}
});
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
messageOutput.add(out);
}
});
return context;
}
}

68
flink/src/test/java/org/apache/zeppelin/flink/FlinkSQLInterpreterTest.java
View file

@ -17,6 +17,10 @@
package org.apache.zeppelin.flink;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
@ -28,11 +32,6 @@ import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
public class FlinkSQLInterpreterTest {
private FlinkInterpreter interpreter;
@ -62,49 +61,48 @@ public class FlinkSQLInterpreterTest {
@Test
public void testSQLInterpreter() throws InterpreterException {
InterpreterResult result = interpreter.interpret(
"val ds = benv.fromElements((1, \"jeff\"), (2, \"andy\"))", getInterpreterContext());
InterpreterResult result =
interpreter.interpret(
"val ds = benv.fromElements((1, \"jeff\"), (2, \"andy\"))", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
result = interpreter.interpret("btenv.registerDataSet(\"table_1\", ds)",
getInterpreterContext());
result =
interpreter.interpret("btenv.registerDataSet(\"table_1\", ds)", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
result = sqlInterpreter.interpret("select * from table_1", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
assertEquals("_1\t_2\n" +
"1\tjeff\n" +
"2\tandy\n", result.message().get(0).getData());
assertEquals("_1\t_2\n" + "1\tjeff\n" + "2\tandy\n", result.message().get(0).getData());
}
private InterpreterContext getInterpreterContext() {
output = "";
InterpreterContext context = InterpreterContext.builder()
.setInterpreterOut(new InterpreterOutput(null))
.setAngularObjectRegistry(new AngularObjectRegistry("flink", null))
.build();
context.out = new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onUpdateAll(InterpreterOutput out) {
InterpreterContext context =
InterpreterContext.builder()
.setInterpreterOut(new InterpreterOutput(null))
.setAngularObjectRegistry(new AngularObjectRegistry("flink", null))
.build();
context.out =
new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onUpdateAll(InterpreterOutput out) {}
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
try {
output = out.toInterpreterResultMessage().getData();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
try {
output = out.toInterpreterResultMessage().getData();
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
messageOutput = out;
}
});
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
messageOutput = out;
}
});
return context;
}
}

7
geode/pom.xml
View file

@ -95,13 +95,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

91
geode/src/main/java/org/apache/zeppelin/geode/GeodeOqlInterpreter.java
View file

@ -4,16 +4,19 @@
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.geode;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.geode.cache.client.ClientCache;
import org.apache.geode.cache.client.ClientCacheFactory;
@ -31,55 +34,36 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
/**
* Apache Geode OQL Interpreter (http://geode.apache.org)
*
* <ul>
* <li>{@code geode.locator.host} - The Geode Locator {@code <HOST>} to connect to.</li>
* <li>{@code geode.locator.port} - The Geode Locator {@code <PORT>} to connect to.</li>
* <li>{@code geode.max.result} - Max number of OQL result to display.</li>
* <li>{@code geode.locator.host} - The Geode Locator {@code <HOST>} to connect to.
* <li>{@code geode.locator.port} - The Geode Locator {@code <PORT>} to connect to.
* <li>{@code geode.max.result} - Max number of OQL result to display.
* </ul>
* <p>
* Sample usages: <br/>
* {@code %geode.oql} <br/>
* {@code SELECT * FROM /regionEmployee e WHERE e.companyId > 95} <br/>
* {@code SELECT * FROM /regionEmployee ORDER BY employeeId} <br/>
* {@code
* SELECT * FROM /regionEmployee
* WHERE companyId IN SET(1, 3, 7) OR lastName IN SET('NameA', 'NameB')
* } <br/>
* {@code
* SELECT e.employeeId, c.id as companyId FROM /regionEmployee e, /regionCompany c
* WHERE e.companyId = c.id
* }
* </p>
* <p>
* OQL specification and sample queries:
*
* <p>Sample usages: <br>
* {@code %geode.oql} <br>
* {@code SELECT * FROM /regionEmployee e WHERE e.companyId > 95} <br>
* {@code SELECT * FROM /regionEmployee ORDER BY employeeId} <br>
* {@code SELECT * FROM /regionEmployee WHERE companyId IN SET(1, 3, 7) OR lastName IN SET('NameA',
* 'NameB') } <br>
* {@code SELECT e.employeeId, c.id as companyId FROM /regionEmployee e, /regionCompany c WHERE
* e.companyId = c.id }
*
* <p>OQL specification and sample queries:
* http://geode-docs.cfapps.io/docs/getting_started/querying_quick_reference.html
* </p>
* <p>
* When the Zeppelin server is collocated with Geode Shell (gfsh) one can use the %sh interpreter to
* run Geode shell commands: <br/>
* {@code
* %sh
* source /etc/geode/conf/geode-env.sh
* gfsh << EOF
* connect --locator=ambari.localdomain[10334]
* destroy region --name=/regionEmployee
* create region --name=regionEmployee --type=REPLICATE
* exit;
* EOF
*}
* </p>
* <p>
* Known issue:http://gemfire.docs.pivotal.io/bugnotes/KnownIssuesGemFire810.html #43673 Using query
* "select * from /exampleRegion.entrySet" fails in a client-server topology and/or in a
*
* <p>When the Zeppelin server is collocated with Geode Shell (gfsh) one can use the %sh interpreter
* to run Geode shell commands: <br>
* {@code %sh source /etc/geode/conf/geode-env.sh gfsh << EOF connect
* --locator=ambari.localdomain[10334] destroy region --name=/regionEmployee create region
* --name=regionEmployee --type=REPLICATE exit; EOF }
*
* <p>Known issue:http://gemfire.docs.pivotal.io/bugnotes/KnownIssuesGemFire810.html #43673 Using
* query "select * from /exampleRegion.entrySet" fails in a client-server topology and/or in a
* PartitionedRegion.
* </p>
*/
public class GeodeOqlInterpreter extends Interpreter {
@ -252,7 +236,6 @@ public class GeodeOqlInterpreter extends Interpreter {
msg.append("" + entry);
}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
logger.info("Run OQL command '{}'", cmd);
@ -276,13 +259,13 @@ public class GeodeOqlInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
GeodeOqlInterpreter.class.getName() + this.hashCode());
return SchedulerFactory.singleton()
.createOrGetFIFOScheduler(GeodeOqlInterpreter.class.getName() + this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return null;
}

69
geode/src/test/java/org/apache/zeppelin/geode/GeodeOqlInterpreterTest.java
View file

@ -4,16 +4,31 @@
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.geode;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Properties;
import org.apache.geode.cache.query.QueryService;
import org.apache.geode.cache.query.SelectResults;
import org.apache.geode.cache.query.Struct;
@ -27,22 +42,6 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class GeodeOqlInterpreterTest {
private static final String OQL_QUERY = "select * from /region";
@ -78,9 +77,9 @@ public class GeodeOqlInterpreterTest {
@Test
public void oqlStructResponse() throws Exception {
String[] fields = new String[]{"field1", "field2"};
Struct s1 = new StructImpl(new StructTypeImpl(fields), new String[]{"val11", "val12"});
Struct s2 = new StructImpl(new StructTypeImpl(fields), new String[]{"val21", "val22"});
String[] fields = new String[] {"field1", "field2"};
Struct s1 = new StructImpl(new StructTypeImpl(fields), new String[] {"val11", "val12"});
Struct s2 = new StructImpl(new StructTypeImpl(fields), new String[] {"val21", "val22"});
testOql(asIterator(s1, s2), "field1\tfield2\t\nval11\tval12\t\nval21\tval22\t\n", 10);
testOql(asIterator(s1, s2), "field1\tfield2\t\nval11\tval12\t\n", 1);
@ -88,8 +87,8 @@ public class GeodeOqlInterpreterTest {
@Test
public void oqlStructResponseWithReservedCharacters() throws Exception {
String[] fields = new String[]{"fi\teld1", "f\nield2"};
Struct s1 = new StructImpl(new StructTypeImpl(fields), new String[]{"v\nal\t1", "val2"});
String[] fields = new String[] {"fi\teld1", "f\nield2"};
Struct s1 = new StructImpl(new StructTypeImpl(fields), new String[] {"v\nal\t1", "val2"});
testOql(asIterator(s1), "fi eld1\tf ield2\t\nv al 1\tval2\t\n", 10);
}
@ -116,8 +115,10 @@ public class GeodeOqlInterpreterTest {
DummyUnspportedType unspported1 = new DummyUnspportedType();
DummyUnspportedType unspported2 = new DummyUnspportedType();
testOql(asIterator(unspported1, unspported2), "Unsuppoted Type\n" + unspported1.toString()
+ "\n" + unspported1.toString() + "\n", 10);
testOql(
asIterator(unspported1, unspported2),
"Unsuppoted Type\n" + unspported1.toString() + "\n" + unspported1.toString() + "\n",
10);
}
private void testOql(Iterator<Object> queryResponseIterator, String expectedOutput, int maxResult)
@ -148,8 +149,8 @@ public class GeodeOqlInterpreterTest {
GeodeOqlInterpreter spyGeodeOqlInterpreter = spy(new GeodeOqlInterpreter(new Properties()));
when(spyGeodeOqlInterpreter.getExceptionOnConnect()).thenReturn(
new RuntimeException("Test Exception On Connect"));
when(spyGeodeOqlInterpreter.getExceptionOnConnect())
.thenReturn(new RuntimeException("Test Exception On Connect"));
InterpreterResult interpreterResult = spyGeodeOqlInterpreter.interpret(OQL_QUERY, null);
@ -162,8 +163,8 @@ public class GeodeOqlInterpreterTest {
GeodeOqlInterpreter spyGeodeOqlInterpreter = spy(new GeodeOqlInterpreter(new Properties()));
when(spyGeodeOqlInterpreter.getQueryService()).thenThrow(
new RuntimeException("Expected Test Exception!"));
when(spyGeodeOqlInterpreter.getQueryService())
.thenThrow(new RuntimeException("Expected Test Exception!"));
InterpreterResult interpreterResult = spyGeodeOqlInterpreter.interpret(OQL_QUERY, null);

11
groovy/pom-groovy-only.xml
View file

@ -79,17 +79,6 @@
</compilerArgs>
</configuration>
</plugin>
<!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
<executions>
</executions>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>

7
groovy/pom.xml
View file

@ -80,13 +80,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

65
groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java
View file

@ -18,6 +18,12 @@ package org.apache.zeppelin.groovy;
import groovy.lang.Closure;
import groovy.xml.MarkupBuilder;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;
import org.apache.thrift.TException;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.display.AngularObject;
@ -27,16 +33,7 @@ import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.slf4j.Logger;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;
/**
* Groovy interpreter for Zeppelin.
*/
/** Groovy interpreter for Zeppelin. */
public class GObject extends groovy.lang.GroovyObjectSupport {
Logger log;
StringWriter out;
@ -45,7 +42,11 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
Map<String, Object> bindings;
GroovyZeppelinContext z;
public GObject(Logger log, StringWriter out, Properties p, InterpreterContext ctx,
public GObject(
Logger log,
StringWriter out,
Properties p,
InterpreterContext ctx,
Map<String, Object> bindings) {
this.log = log;
this.out = out;
@ -83,9 +84,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
}
}
/**
* returns gui object.
*/
/** returns gui object. */
public GUI getGui() {
return z.getGui();
}
@ -125,12 +124,11 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
}
@ZeppelinApi
public Collection<Object> checkbox(String name, Collection<Object> defaultChecked,
Map<Object, String> options) {
public Collection<Object> checkbox(
String name, Collection<Object> defaultChecked, Map<Object, String> options) {
return z.checkbox(name, new ArrayList<Object>(defaultChecked), toParamOptions(options));
}
/**
* Returns shared variable if it was previously set. The same as getting groovy script variables
* but this method will return null if script variable not assigned. To understand groovy script
@ -155,9 +153,9 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
}
/**
* Sets a new value to interpreter's shared variables.
* Could be set by <code>put('varName', newValue )</code>
* or by just assigning <code>varName = value</code> without declaring a variable.
* Sets a new value to interpreter's shared variables. Could be set by <code>
* put('varName', newValue )</code> or by just assigning <code>varName = value</code> without
* declaring a variable.
*/
public Object put(String varName, Object newValue) {
return bindings.put(varName, newValue);
@ -165,6 +163,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
/**
* starts or continues rendering html/angular and returns MarkupBuilder to build html.
*
* <pre> g.html().with{
* h1("hello")
* h2("world")
@ -187,7 +186,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
StringBuffer sb = out.getBuffer();
startOutputType("%table");
if (obj instanceof groovy.lang.Closure) {
//if closure run and get result collection
// if closure run and get result collection
obj = ((Closure) obj).call();
}
if (obj instanceof Collection) {
@ -245,8 +244,8 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
}
/**
* Create angular variable in notebook scope and bind with front end Angular display system.
* If variable exists, it'll be overwritten.
* Create angular variable in notebook scope and bind with front end Angular display system. If
* variable exists, it'll be overwritten.
*
* @param name name of the variable
* @param o value
@ -255,25 +254,19 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
angularBind(name, o, interpreterContext.getNoteId());
}
/**
* Run paragraph by id.
*/
/** Run paragraph by id. */
@ZeppelinApi
public void run(String noteId, String paragraphId) throws IOException {
z.run(noteId, paragraphId);
}
/**
* Run paragraph by id.
*/
/** Run paragraph by id. */
@ZeppelinApi
public void run(String paragraphId) throws IOException {
z.run(paragraphId);
}
/**
* Run paragraph by id.
*/
/** Run paragraph by id. */
@ZeppelinApi
public void run(String noteId, String paragraphId, InterpreterContext context)
throws IOException {
@ -288,17 +281,13 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
z.runNote(noteId, context);
}
/**
* Run all paragraphs. except this.
*/
/** Run all paragraphs. except this. */
@ZeppelinApi
public void runAll() throws IOException {
z.runAll(interpreterContext);
}
/**
* Run all paragraphs. except this.
*/
/** Run all paragraphs. except this. */
@ZeppelinApi
public void runAll(InterpreterContext context) throws IOException {
z.runNote(context.getNoteId());

60
groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java
View file

@ -17,12 +17,8 @@
package org.apache.zeppelin.groovy;
import org.codehaus.groovy.control.CompilerConfiguration;
import org.codehaus.groovy.runtime.StackTraceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import groovy.lang.GroovyShell;
import groovy.lang.Script;
import java.io.File;
import java.io.PrintWriter;
import java.io.StringWriter;
@ -35,10 +31,6 @@ import java.util.Properties;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.ConcurrentHashMap;
import groovy.lang.GroovyShell;
import groovy.lang.Script;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -47,18 +39,20 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.codehaus.groovy.control.CompilerConfiguration;
import org.codehaus.groovy.runtime.StackTraceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Groovy interpreter for Zeppelin.
*/
/** Groovy interpreter for Zeppelin. */
public class GroovyInterpreter extends Interpreter {
Logger log = LoggerFactory.getLogger(GroovyInterpreter.class);
GroovyShell shell = null; //new GroovyShell();
//here we will store Interpreters shared variables. concurrent just in case.
GroovyShell shell = null; // new GroovyShell();
// here we will store Interpreters shared variables. concurrent just in case.
Map<String, Object> sharedBindings = new ConcurrentHashMap<String, Object>();
//cache for groovy compiled scripts
Map<String, Class<Script>> scriptCache = Collections
.synchronizedMap(new WeakHashMap<String, Class<Script>>(100));
// cache for groovy compiled scripts
Map<String, Class<Script>> scriptCache =
Collections.synchronizedMap(new WeakHashMap<String, Class<Script>>(100));
public GroovyInterpreter(Properties property) {
super(property);
@ -72,9 +66,14 @@ public class GroovyInterpreter extends Interpreter {
String classes = getProperty("GROOVY_CLASSES");
if (classes == null || classes.length() == 0) {
try {
File jar = new File(
GroovyInterpreter.class.getProtectionDomain().getCodeSource().getLocation().toURI()
.getPath());
File jar =
new File(
GroovyInterpreter.class
.getProtectionDomain()
.getCodeSource()
.getLocation()
.toURI()
.getPath());
classes = new File(jar.getParentFile(), "classes").toString();
} catch (Exception e) {
log.error(e.getMessage());
@ -123,8 +122,8 @@ public class GroovyInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return null;
}
@ -161,23 +160,24 @@ public class GroovyInterpreter extends Interpreter {
try {
Script script = getGroovyScript(contextInterpreter.getParagraphId(), cmd);
Job runningJob = getRunningJob(contextInterpreter.getParagraphId());
runningJob.info()
.put("CURRENT_THREAD", Thread.currentThread()); //to be able to terminate thread
runningJob
.info()
.put("CURRENT_THREAD", Thread.currentThread()); // to be able to terminate thread
Map<String, Object> bindings = script.getBinding().getVariables();
bindings.clear();
StringWriter out = new StringWriter((int) (cmd.length() * 1.75));
//put shared bindings evaluated in this interpreter
// put shared bindings evaluated in this interpreter
bindings.putAll(sharedBindings);
//put predefined bindings
// put predefined bindings
bindings.put("g", new GObject(log, out, this.getProperties(), contextInterpreter, bindings));
bindings.put("out", new PrintWriter(out, true));
script.run();
//let's get shared variables defined in current script and store them in shared map
// let's get shared variables defined in current script and store them in shared map
for (Map.Entry<String, Object> e : bindings.entrySet()) {
if (!predefinedBindings.contains(e.getKey())) {
if (log.isTraceEnabled()) {
log.trace("groovy script variable " + e); //let's see what we have...
log.trace("groovy script variable " + e); // let's see what we have...
}
sharedBindings.put(e.getKey(), e.getValue());
}
@ -205,7 +205,7 @@ public class GroovyInterpreter extends Interpreter {
Thread t = (Thread) object;
t.dumpStack();
t.interrupt();
//t.stop(); //TODO(dlukyanov): need some way to terminate maybe through GObject..
// t.stop(); //TODO(dlukyanov): need some way to terminate maybe through GObject..
} catch (Throwable t) {
log.error("Failed to cancel script: " + t, t);
}

9
groovy/src/main/java/org/apache/zeppelin/groovy/GroovyZeppelinContext.java
View file

@ -17,15 +17,12 @@
package org.apache.zeppelin.groovy;
import java.util.List;
import java.util.Map;
import org.apache.zeppelin.interpreter.BaseZeppelinContext;
import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
import java.util.List;
import java.util.Map;
/**
* ZeppelinContext for Groovy
*/
/** ZeppelinContext for Groovy */
public class GroovyZeppelinContext extends BaseZeppelinContext {
public GroovyZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {

7
hbase/pom.xml
View file

@ -124,13 +124,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

52
hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java
View file

@ -14,11 +14,6 @@
package org.apache.zeppelin.hbase;
import org.jruby.embed.LocalContextScope;
import org.jruby.embed.ScriptingContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@ -27,7 +22,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
@ -35,22 +29,25 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.jruby.embed.LocalContextScope;
import org.jruby.embed.ScriptingContainer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Support for HBase Shell. All the commands documented here
* http://hbase.apache.org/book.html#shell is supported.
* Support for HBase Shell. All the commands documented here http://hbase.apache.org/book.html#shell
* is supported.
*
* Requirements:
* HBase Shell should be installed on the same machine. To be more specific, the following dir.
* should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby
* HBase Shell should be able to connect to the HBase cluster from terminal. This makes sure
* that the client is configured properly.
* <p>Requirements: HBase Shell should be installed on the same machine. To be more specific, the
* following dir. should be available:
* https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby HBase Shell should be able
* to connect to the HBase cluster from terminal. This makes sure that the client is configured
* properly.
*
* The interpreter takes 3 config parameters:
* hbase.home: Root directory where HBase is installed. Default is /usr/lib/hbase/
* hbase.ruby.sources: Dir where shell ruby code is installed.
* Path is relative to hbase.home. Default: lib/ruby
* zeppelin.hbase.test.mode: (Testing only) Disable checks for unit and manual tests. Default: false
* <p>The interpreter takes 3 config parameters: hbase.home: Root directory where HBase is
* installed. Default is /usr/lib/hbase/ hbase.ruby.sources: Dir where shell ruby code is installed.
* Path is relative to hbase.home. Default: lib/ruby zeppelin.hbase.test.mode: (Testing only)
* Disable checks for unit and manual tests. Default: false
*/
public class HbaseInterpreter extends Interpreter {
public static final String HBASE_HOME = "hbase.home";
@ -68,7 +65,7 @@ public class HbaseInterpreter extends Interpreter {
@Override
public void open() throws InterpreterException {
this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON);
this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON);
this.writer = new StringWriter();
scriptingContainer.setOutput(this.writer);
@ -82,8 +79,8 @@ public class HbaseInterpreter extends Interpreter {
File f = absRubySrc.toFile();
if (!f.exists() || !f.isDirectory()) {
throw new InterpreterException("HBase ruby sources is not available at '" + absRubySrc
+ "'");
throw new InterpreterException(
"HBase ruby sources is not available at '" + absRubySrc + "'");
}
logger.info("Absolute Ruby Source:" + absRubySrc.toString());
@ -139,20 +136,17 @@ public class HbaseInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
HbaseInterpreter.class.getName() + this.hashCode());
return SchedulerFactory.singleton()
.createOrGetFIFOScheduler(HbaseInterpreter.class.getName() + this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return null;
}
private static String getSystemDefault(
String envName,
String propertyName,
String defaultValue) {
private static String getSystemDefault(String envName, String propertyName, String defaultValue) {
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);

25
hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java
View file

@ -18,20 +18,16 @@ import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.notNullValue;
import static org.junit.Assert.assertEquals;
import java.util.Properties;
import org.apache.log4j.BasicConfigurator;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
/**
* Tests for HBase Interpreter.
*/
/** Tests for HBase Interpreter. */
public class HbaseInterpreterTest {
private static Logger logger = LoggerFactory.getLogger(HbaseInterpreterTest.class);
private static HbaseInterpreter hbaseInterpreter;
@ -47,7 +43,7 @@ public class HbaseInterpreterTest {
hbaseInterpreter = new HbaseInterpreter(properties);
hbaseInterpreter.open();
}
@Test
public void newObject() {
assertThat(hbaseInterpreter, notNullValue());
@ -60,10 +56,10 @@ public class HbaseInterpreterTest {
assertEquals(result.message().get(0).getType(), InterpreterResult.Type.TEXT);
assertEquals("Hello World\n", result.message().get(0).getData());
}
public void putsLoadPath() {
InterpreterResult result = hbaseInterpreter.interpret(
"require 'two_power'; puts twoToThePowerOf(4)", null);
InterpreterResult result =
hbaseInterpreter.interpret("require 'two_power'; puts twoToThePowerOf(4)", null);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(result.message().get(0).getType(), InterpreterResult.Type.TEXT);
assertEquals("16\n", result.message().get(0).getData());
@ -73,7 +69,8 @@ public class HbaseInterpreterTest {
public void testException() {
InterpreterResult result = hbaseInterpreter.interpret("plot practical joke", null);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals("(NameError) undefined local variable or method `joke' for main:Object",
result.message().get(0).getData());
assertEquals(
"(NameError) undefined local variable or method `joke' for main:Object",
result.message().get(0).getData());
}
}

27
helium-dev/src/main/java/org/apache/zeppelin/helium/DevInterpreter.java
View file

@ -17,20 +17,17 @@
package org.apache.zeppelin.helium;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
/**
* Dummy interpreter to support development mode for Zeppelin app
*/
/** Dummy interpreter to support development mode for Zeppelin app */
public class DevInterpreter extends Interpreter {
private InterpreterEvent interpreterEvent;
@ -41,9 +38,7 @@ public class DevInterpreter extends Interpreter {
return replName.equals("dev");
}
/**
* event handler for org.apache.zeppelin.helium.ZeppelinApplicationDevServer
*/
/** event handler for org.apache.zeppelin.helium.ZeppelinApplicationDevServer */
public static interface InterpreterEvent {
public InterpreterResult interpret(String st, InterpreterContext context);
}
@ -63,8 +58,7 @@ public class DevInterpreter extends Interpreter {
}
@Override
public void close() {
}
public void close() {}
public void rerun() {
try {
@ -87,8 +81,7 @@ public class DevInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {
}
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
@ -101,8 +94,8 @@ public class DevInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return new LinkedList<>();
}

10
helium-dev/src/main/java/org/apache/zeppelin/helium/DevZeppelinContext.java
View file

@ -15,18 +15,14 @@
* limitations under the License.
*/
package org.apache.zeppelin.helium;
import org.apache.zeppelin.interpreter.BaseZeppelinContext;
import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
import java.util.List;
import java.util.Map;
import org.apache.zeppelin.interpreter.BaseZeppelinContext;
import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
/**
* ZeppelinContext for DevInterpreter
*/
/** ZeppelinContext for DevInterpreter */
public class DevZeppelinContext extends BaseZeppelinContext {
public DevZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
super(hooks, maxResult);

63
helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinApplicationDevServer.java
View file

@ -17,10 +17,9 @@
package org.apache.zeppelin.helium;
import com.google.gson.Gson;
import java.io.IOException;
import java.lang.reflect.Constructor;
import com.google.gson.Gson;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Level;
import org.apache.log4j.PatternLayout;
@ -31,9 +30,7 @@ import org.apache.zeppelin.resource.ResourceSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Run this server for development mode.
*/
/** Run this server for development mode. */
public class ZeppelinApplicationDevServer extends ZeppelinDevServer {
final Logger logger = LoggerFactory.getLogger(ZeppelinApplicationDevServer.class);
@ -42,13 +39,13 @@ public class ZeppelinApplicationDevServer extends ZeppelinDevServer {
private Application app;
private InterpreterOutput out;
public ZeppelinApplicationDevServer(final String className, ResourceSet resourceSet) throws
Exception {
public ZeppelinApplicationDevServer(final String className, ResourceSet resourceSet)
throws Exception {
this(Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT, className, resourceSet);
}
public ZeppelinApplicationDevServer(int port, String className, ResourceSet resourceSet) throws
Exception {
public ZeppelinApplicationDevServer(int port, String className, ResourceSet resourceSet)
throws Exception {
super(port);
this.className = className;
this.resourceSet = resourceSet;
@ -56,17 +53,16 @@ public class ZeppelinApplicationDevServer extends ZeppelinDevServer {
};
void setLogger() {
ConsoleAppender console = new ConsoleAppender(); //create appender
//configure the appender
ConsoleAppender console = new ConsoleAppender(); // create appender
// configure the appender
String PATTERN = "%d [%p|%c|%C{1}] %m%n";
console.setLayout(new PatternLayout(PATTERN));
console.setThreshold(Level.DEBUG);
console.activateOptions();
//add appender to any Logger (here is root)
// add appender to any Logger (here is root)
org.apache.log4j.Logger.getRootLogger().addAppender(console);
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
if (app == null) {
@ -140,28 +136,29 @@ public class ZeppelinApplicationDevServer extends ZeppelinDevServer {
if (out == null) {
final RemoteInterpreterEventClient eventClient = getIntpEventClient();
try {
out = new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onUpdateAll(InterpreterOutput out) {
out =
new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onUpdateAll(InterpreterOutput out) {}
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
eventClient.onInterpreterOutputAppend(
noteId, paragraphId, index, new String(line));
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
eventClient.onInterpreterOutputAppend(noteId, paragraphId, index, new String(line));
}
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
try {
eventClient.onInterpreterOutputUpdate(noteId, paragraphId,
index, out.getType(), new String(out.toByteArray()));
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}, this);
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
try {
eventClient.onInterpreterOutputUpdate(
noteId, paragraphId, index, out.getType(), new String(out.toByteArray()));
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
},
this);
} catch (IOException e) {
return null;
}

57
helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
View file

@ -20,7 +20,6 @@ package org.apache.zeppelin.helium;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import org.apache.thrift.TException;
import org.apache.zeppelin.helium.DevInterpreter.InterpreterEvent;
import org.apache.zeppelin.interpreter.*;
@ -29,15 +28,14 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Interpreter development server
*/
public class ZeppelinDevServer extends
RemoteInterpreterServer implements InterpreterEvent, InterpreterOutputChangeListener {
/** Interpreter development server */
public class ZeppelinDevServer extends RemoteInterpreterServer
implements InterpreterEvent, InterpreterOutputChangeListener {
private static final Logger logger = LoggerFactory.getLogger(ZeppelinDevServer.class);
private DevInterpreter interpreter = null;
private InterpreterOutput out;
public ZeppelinDevServer(int port) throws TException, IOException {
super(null, port, null, ":");
}
@ -58,8 +56,7 @@ public class ZeppelinDevServer extends
}
Interpreter intp = super.getInterpreter(sessionId, className);
interpreter = (DevInterpreter) (
((LazyOpenInterpreter) intp).getInnerInterpreter());
interpreter = (DevInterpreter) (((LazyOpenInterpreter) intp).getInnerInterpreter());
interpreter.setInterpreterEvent(this);
return super.getInterpreter(sessionId, className);
}
@ -70,27 +67,29 @@ public class ZeppelinDevServer extends
if (out == null) {
final RemoteInterpreterEventClient eventClient = getIntpEventClient();
try {
out = new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onUpdateAll(InterpreterOutput out) {
out =
new InterpreterOutput(
new InterpreterOutputListener() {
@Override
public void onUpdateAll(InterpreterOutput out) {}
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
eventClient.onInterpreterOutputAppend(
noteId, paragraphId, index, new String(line));
}
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
eventClient.onInterpreterOutputAppend(noteId, paragraphId, index, new String(line));
}
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
try {
eventClient.onInterpreterOutputUpdate(noteId, paragraphId,
index, out.getType(), new String(out.toByteArray()));
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}, this);
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
try {
eventClient.onInterpreterOutputUpdate(
noteId, paragraphId, index, out.getType(), new String(out.toByteArray()));
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
},
this);
} catch (IOException e) {
return null;
}
@ -115,9 +114,7 @@ public class ZeppelinDevServer extends
interpreter.rerun();
}
/**
* Wait until %dev paragraph is executed and connected to this process
*/
/** Wait until %dev paragraph is executed and connected to this process */
public void waitForConnected() {
synchronized (this) {
while (!isConnected()) {

7
ignite/pom.xml
View file

@ -114,13 +114,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>

83
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java
View file

@ -16,14 +16,6 @@
*/
package org.apache.zeppelin.ignite;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintWriter;
@ -35,7 +27,21 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Console;
import scala.Some;
import scala.collection.JavaConversions;
@ -45,29 +51,20 @@ import scala.tools.nsc.interpreter.Results.Result;
import scala.tools.nsc.settings.MutableSettings.BooleanSetting;
import scala.tools.nsc.settings.MutableSettings.PathSetting;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
/**
* Apache Ignite interpreter (http://ignite.incubator.apache.org/).
*
* Use the following properties for interpreter configuration:
* <p>Use the following properties for interpreter configuration:
*
* <ul>
* <li>{@code ignite.addresses} - coma separated list of hosts in form {@code <host>:<port>}
* or {@code <host>:<port_1>..<port_n>} </li>
* <li>{@code ignite.clientMode} - indicates that Ignite interpreter
* should start node in client mode ({@code true} or {@code false}).</li>
* <li>{@code ignite.peerClassLoadingEnabled} - enables/disables peer class loading
* ({@code true} or {@code false}).</li>
* <li>{@code ignite.config.url} - URL for Ignite configuration. If this URL specified then
* all aforementioned properties will not be taken in account.</li>
* <li>{@code ignite.addresses} - coma separated list of hosts in form {@code <host>:<port>} or
* {@code <host>:<port_1>..<port_n>}
* <li>{@code ignite.clientMode} - indicates that Ignite interpreter should start node in client
* mode ({@code true} or {@code false}).
* <li>{@code ignite.peerClassLoadingEnabled} - enables/disables peer class loading ({@code true}
* or {@code false}).
* <li>{@code ignite.config.url} - URL for Ignite configuration. If this URL specified then all
* aforementioned properties will not be taken in account.
* </ul>
*/
public class IgniteInterpreter extends Interpreter {
@ -162,9 +159,11 @@ public class IgniteInterpreter extends Interpreter {
}
public Object getLastObject() {
Object obj = imain.lastRequest().lineRep().call(
"$result",
JavaConversions.asScalaBuffer(new LinkedList<>()));
Object obj =
imain
.lastRequest()
.lineRep()
.call("$result", JavaConversions.asScalaBuffer(new LinkedList<>()));
return obj;
}
@ -188,14 +187,14 @@ public class IgniteInterpreter extends Interpreter {
conf.setDiscoverySpi(discoSpi);
conf.setPeerClassLoadingEnabled(
Boolean.parseBoolean(getProperty(IGNITE_PEER_CLASS_LOADING_ENABLED)));
Boolean.parseBoolean(getProperty(IGNITE_PEER_CLASS_LOADING_ENABLED)));
ignite = Ignition.start(conf);
}
initEx = null;
} catch (Exception e) {
logger.error("Error in IgniteInterpreter while getIgnite: " , e);
logger.error("Error in IgniteInterpreter while getIgnite: ", e);
initEx = e;
}
}
@ -211,9 +210,10 @@ public class IgniteInterpreter extends Interpreter {
if (getIgnite() != null) {
binder.put("ignite", ignite);
imain.interpret("@transient val ignite = "
+ "_binder.get(\"ignite\")"
+ ".asInstanceOf[org.apache.ignite.Ignite]");
imain.interpret(
"@transient val ignite = "
+ "_binder.get(\"ignite\")"
+ ".asInstanceOf[org.apache.ignite.Ignite]");
}
} finally {
Thread.currentThread().setContextClassLoader(contextClassLoader);
@ -263,8 +263,7 @@ public class IgniteInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {
}
public void cancel(InterpreterContext context) {}
private InterpreterResult interpret(String[] lines) {
String[] linesToRun = new String[lines.length + 1];
@ -277,7 +276,7 @@ public class IgniteInterpreter extends Interpreter {
String incomplete = "";
for (int l = 0; l < linesToRun.length; l++) {
String s = linesToRun[l];
String s = linesToRun[l];
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
if (l + 1 < linesToRun.length) {
String nextLine = linesToRun[l + 1].trim();
@ -334,14 +333,14 @@ public class IgniteInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return new LinkedList<>();
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
IgniteInterpreter.class.getName() + this.hashCode());
return SchedulerFactory.singleton()
.createOrGetFIFOScheduler(IgniteInterpreter.class.getName() + this.hashCode());
}
}

5
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreterUtils.java
View file

@ -19,12 +19,11 @@ package org.apache.zeppelin.ignite;
import org.apache.zeppelin.interpreter.InterpreterResult;
/**
* Apache Ignite interpreter utils.
*/
/** Apache Ignite interpreter utils. */
public class IgniteInterpreterUtils {
/**
* Builds error result from given exception.
*
* @param e Exception.
* @return result.
*/

27
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteSqlInterpreter.java
View file

@ -16,9 +16,6 @@
*/
package org.apache.zeppelin.ignite;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@ -28,7 +25,6 @@ import java.sql.Statement;
import java.util.LinkedList;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
@ -37,19 +33,20 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Apache Ignite SQL interpreter (http://ignite.incubator.apache.org/).
*
* Use {@code ignite.jdbc.url} property to set up JDBC connection URL.
* URL has the following pattern:
* {@code jdbc:ignite://<hostname>:<port>/<cache_name>}
* <p>Use {@code ignite.jdbc.url} property to set up JDBC connection URL. URL has the following
* pattern: {@code jdbc:ignite://<hostname>:<port>/<cache_name>}
*
* <ul>
* <li>Hostname is required.</li>
* <li>If port is not defined, 11211 is used (default for Ignite client).</li>
* <li>Leave cache_name empty if you are connecting to a default cache.
* Note that the cache name is case sensitive.</li>
* <li>Hostname is required.
* <li>If port is not defined, 11211 is used (default for Ignite client).
* <li>Leave cache_name empty if you are connecting to a default cache. Note that the cache name
* is case sensitive.
* </ul>
*/
public class IgniteSqlInterpreter extends Interpreter {
@ -179,13 +176,13 @@ public class IgniteSqlInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
IgniteSqlInterpreter.class.getName() + this.hashCode());
return SchedulerFactory.singleton()
.createOrGetFIFOScheduler(IgniteSqlInterpreter.class.getName() + this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return new LinkedList<>();
}
}

37
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java
View file

@ -16,6 +16,11 @@
*/
package org.apache.zeppelin.ignite;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.Collections;
import java.util.Properties;
import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.configuration.IgniteConfiguration;
@ -27,15 +32,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.Collections;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* Tests for Apache Ignite interpreter ({@link IgniteInterpreter}).
*/
/** Tests for Apache Ignite interpreter ({@link IgniteInterpreter}). */
public class IgniteInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
@ -60,8 +57,9 @@ public class IgniteInterpreterTest {
ignite = Ignition.start(cfg);
Properties props = new Properties();
props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL,
"jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml");
props.setProperty(
IgniteSqlInterpreter.IGNITE_JDBC_URL,
"jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml");
props.setProperty(IgniteInterpreter.IGNITE_CLIENT_MODE, "false");
props.setProperty(IgniteInterpreter.IGNITE_PEER_CLASS_LOADING_ENABLED, "false");
props.setProperty(IgniteInterpreter.IGNITE_ADDRESSES, HOST);
@ -80,12 +78,21 @@ public class IgniteInterpreterTest {
public void testInterpret() {
String sizeVal = "size";
InterpreterResult result = intp.interpret("import org.apache.ignite.IgniteCache\n" +
"val " + sizeVal + " = ignite.cluster().nodes().size()", INTP_CONTEXT);
InterpreterResult result =
intp.interpret(
"import org.apache.ignite.IgniteCache\n"
+ "val "
+ sizeVal
+ " = ignite.cluster().nodes().size()",
INTP_CONTEXT);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertTrue(result.message().get(0).getData().contains(sizeVal + ": Int = " +
ignite.cluster().nodes().size()));
assertTrue(
result
.message()
.get(0)
.getData()
.contains(sizeVal + ": Int = " + ignite.cluster().nodes().size()));
result = intp.interpret("\"123\"\n .toInt", INTP_CONTEXT);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());

22
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java
View file

@ -16,6 +16,10 @@
*/
package org.apache.zeppelin.ignite;
import static org.junit.Assert.assertEquals;
import java.util.Collections;
import java.util.Properties;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
@ -32,14 +36,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.Collections;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
/**
* Tests for Apache Ignite SQL interpreter ({@link IgniteSqlInterpreter}).
*/
/** Tests for Apache Ignite SQL interpreter ({@link IgniteSqlInterpreter}). */
public class IgniteSqlInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
@ -65,8 +62,9 @@ public class IgniteSqlInterpreterTest {
ignite = Ignition.start(cfg);
Properties props = new Properties();
props.setProperty(IgniteSqlInterpreter.IGNITE_JDBC_URL,
"jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml");
props.setProperty(
IgniteSqlInterpreter.IGNITE_JDBC_URL,
"jdbc:ignite:cfg://cache=person@default-ignite-jdbc.xml");
intp = new IgniteSqlInterpreter(props);
@ -90,8 +88,8 @@ public class IgniteSqlInterpreterTest {
@Test
public void testSql() {
InterpreterResult result = intp.interpret("select name, age from person where age > 10",
INTP_CONTEXT);
InterpreterResult result =
intp.interpret("select name, age from person where age > 10", INTP_CONTEXT);
assertEquals(Code.SUCCESS, result.code());
assertEquals(Type.TABLE, result.message().get(0).getType());

9
ignite/src/test/java/org/apache/zeppelin/ignite/Person.java
View file

@ -16,16 +16,13 @@
*/
package org.apache.zeppelin.ignite;
import java.io.Serializable;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import java.io.Serializable;
public class Person implements Serializable {
@QuerySqlField
private String name;
@QuerySqlField private String name;
@QuerySqlField
private int age;
@QuerySqlField private int age;
public Person(String name, int age) {
this.name = name;

7
java/pom.xml
View file

@ -76,13 +76,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

30
java/src/main/java/org/apache/zeppelin/java/JavaInterpreter.java
View file

@ -17,6 +17,11 @@
package org.apache.zeppelin.java;
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -24,15 +29,7 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
/**
* Java interpreter
*/
/** Java interpreter */
public class JavaInterpreter extends Interpreter {
private static final Logger logger = LoggerFactory.getLogger(JavaInterpreter.class);
@ -42,9 +39,7 @@ public class JavaInterpreter extends Interpreter {
}
@Override
public void open() {
}
public void open() {}
@Override
public void close() {
@ -71,15 +66,11 @@ public class JavaInterpreter extends Interpreter {
} catch (Exception e) {
logger.error("Exception in Interpreter while interpret", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {
}
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
@ -92,9 +83,8 @@ public class JavaInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return Collections.emptyList();
}
}

12
java/src/main/java/org/apache/zeppelin/java/JavaInterpreterUtils.java
View file

@ -20,25 +20,25 @@ package org.apache.zeppelin.java;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Java interpreter utility methods
*/
/** Java interpreter utility methods */
public class JavaInterpreterUtils {
/**
* Convert a map to %table display system to leverage Zeppelin's built in visualization
*
* @param keyName Key column name
* @param valueName Value column name
* @param rows Map of keys and values
* @return Zeppelin %table
*/
public static String displayTableFromSimpleMap(String keyName, String valueName, Map<?, ?> rows){
public static String displayTableFromSimpleMap(String keyName, String valueName, Map<?, ?> rows) {
String table = "%table\n";
table += keyName + "\t" + valueName + "\n";
table += rows.entrySet().stream()
table +=
rows.entrySet()
.stream()
.map(e -> e.getKey() + "\t" + e.getValue())
.collect(Collectors.joining("\n"));
return table;
}
}

50
java/src/main/java/org/apache/zeppelin/java/StaticRepl.java
View file

@ -20,16 +20,6 @@ package org.apache.zeppelin.java;
import com.thoughtworks.qdox.JavaProjectBuilder;
import com.thoughtworks.qdox.model.JavaClass;
import com.thoughtworks.qdox.model.JavaSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.tools.Diagnostic;
import javax.tools.DiagnosticCollector;
import javax.tools.JavaCompiler;
import javax.tools.JavaCompiler.CompilationTask;
import javax.tools.JavaFileObject;
import javax.tools.SimpleJavaFileObject;
import javax.tools.ToolProvider;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
@ -40,10 +30,17 @@ import java.net.URL;
import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.List;
import javax.tools.Diagnostic;
import javax.tools.DiagnosticCollector;
import javax.tools.JavaCompiler;
import javax.tools.JavaCompiler.CompilationTask;
import javax.tools.JavaFileObject;
import javax.tools.SimpleJavaFileObject;
import javax.tools.ToolProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* StaticRepl for compling the java code in memory
*/
/** StaticRepl for compling the java code in memory */
public class StaticRepl {
static Logger logger = LoggerFactory.getLogger(StaticRepl.class);
@ -65,8 +62,8 @@ public class StaticRepl {
boolean hasMain = false;
for (int j = 0; j < classes.get(i).getMethods().size(); j++) {
if (classes.get(i).getMethods().get(j).getName().equals("main") && classes.get(i)
.getMethods().get(j).isStatic()) {
if (classes.get(i).getMethods().get(j).getName().equals("main")
&& classes.get(i).getMethods().get(j).isStatic()) {
mainClassName = classes.get(i).getName();
hasMain = true;
break;
@ -75,13 +72,12 @@ public class StaticRepl {
if (hasMain == true) {
break;
}
}
// if there isn't Main method, will retuen error
if (mainClassName == null) {
logger.error("Exception for Main method", "There isn't any class "
+ "containing static main method.");
logger.error(
"Exception for Main method", "There isn't any class " + "containing static main method.");
throw new Exception("There isn't any class containing static main method.");
}
@ -115,8 +111,8 @@ public class StaticRepl {
if (diagnostic.getLineNumber() == -1) {
continue;
}
System.err.println("line " + diagnostic.getLineNumber() + " : "
+ diagnostic.getMessage(null));
System.err.println(
"line " + diagnostic.getLineNumber() + " : " + diagnostic.getMessage(null));
}
System.out.flush();
System.err.flush();
@ -129,12 +125,12 @@ public class StaticRepl {
try {
// creating new class loader
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[]{new File("").toURI()
.toURL()});
URLClassLoader classLoader =
URLClassLoader.newInstance(new URL[] {new File("").toURI().toURL()});
// execute the Main method
Class.forName(generatedClassName, true, classLoader)
.getDeclaredMethod("main", new Class[]{String[].class})
.invoke(null, new Object[]{null});
.getDeclaredMethod("main", new Class[] {String[].class})
.invoke(null, new Object[] {null});
System.out.flush();
System.err.flush();
@ -145,7 +141,9 @@ public class StaticRepl {
return baosOut.toString();
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException
} catch (ClassNotFoundException
| NoSuchMethodException
| IllegalAccessException
| InvocationTargetException e) {
logger.error("Exception in Interpreter while execution", e);
System.err.println(e);
@ -161,9 +159,7 @@ public class StaticRepl {
System.setErr(oldErr);
}
}
}
}
class JavaSourceFromString extends SimpleJavaFileObject {

7
jdbc/pom.xml
View file

@ -362,13 +362,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

290
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -5,12 +5,12 @@
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jdbc;
@ -19,22 +19,6 @@ import static org.apache.commons.lang.StringUtils.isEmpty;
import static org.apache.commons.lang.StringUtils.isNotEmpty;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import org.apache.commons.dbcp2.ConnectionFactory;
import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
import org.apache.commons.dbcp2.PoolableConnectionFactory;
import org.apache.commons.dbcp2.PoolingDriver;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.sql.Connection;
@ -54,7 +38,19 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.commons.dbcp2.ConnectionFactory;
import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
import org.apache.commons.dbcp2.PoolableConnectionFactory;
import org.apache.commons.dbcp2.PoolingDriver;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.mutable.MutableBoolean;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -67,28 +63,24 @@ import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.user.UserCredentials;
import org.apache.zeppelin.user.UsernamePassword;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* JDBC interpreter for Zeppelin. This interpreter can also be used for accessing HAWQ,
* GreenplumDB, MariaDB, MySQL, Postgres and Redshift.
* JDBC interpreter for Zeppelin. This interpreter can also be used for accessing HAWQ, GreenplumDB,
* MariaDB, MySQL, Postgres and Redshift.
*
* <ul>
* <li>{@code default.url} - JDBC URL to connect to.</li>
* <li>{@code default.user} - JDBC user name..</li>
* <li>{@code default.password} - JDBC password..</li>
* <li>{@code default.driver.name} - JDBC driver name.</li>
* <li>{@code common.max.result} - Max number of SQL result to display.</li>
* <li>{@code default.url} - JDBC URL to connect to.
* <li>{@code default.user} - JDBC user name..
* <li>{@code default.password} - JDBC password..
* <li>{@code default.driver.name} - JDBC driver name.
* <li>{@code common.max.result} - Max number of SQL result to display.
* </ul>
*
* <p>
* How to use: <br/>
* {@code %jdbc.sql} <br/>
* {@code
* SELECT store_id, count(*)
* FROM retail_demo.order_lineitems_pxf
* GROUP BY store_id;
* }
* </p>
* <p>How to use: <br>
* {@code %jdbc.sql} <br>
* {@code SELECT store_id, count(*) FROM retail_demo.order_lineitems_pxf GROUP BY store_id; }
*/
public class JDBCInterpreter extends KerberosInterpreter {
private Logger logger = LoggerFactory.getLogger(JDBCInterpreter.class);
@ -134,7 +126,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
private static final String CONCURRENT_EXECUTION_KEY = "zeppelin.jdbc.concurrent.use";
private static final String CONCURRENT_EXECUTION_COUNT =
"zeppelin.jdbc.concurrent.max_connection";
"zeppelin.jdbc.concurrent.max_connection";
private static final String DBCP_STRING = "jdbc:apache:commons:dbcp:";
private final HashMap<String, Properties> basePropretiesMap;
@ -196,8 +188,13 @@ public class JDBCInterpreter extends KerberosInterpreter {
if (!COMMON_KEY.equals(key)) {
Properties properties = basePropretiesMap.get(key);
if (!properties.containsKey(DRIVER_KEY) || !properties.containsKey(URL_KEY)) {
logger.error("{} will be ignored. {}.{} and {}.{} is mandatory.",
key, DRIVER_KEY, key, key, URL_KEY);
logger.error(
"{} will be ignored. {}.{} and {}.{} is mandatory.",
key,
DRIVER_KEY,
key,
key,
URL_KEY);
removeKeySet.add(key);
}
}
@ -222,20 +219,24 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
private void setMaxLineResults() {
if (basePropretiesMap.containsKey(COMMON_KEY) &&
basePropretiesMap.get(COMMON_KEY).containsKey(MAX_LINE_KEY)) {
if (basePropretiesMap.containsKey(COMMON_KEY)
&& basePropretiesMap.get(COMMON_KEY).containsKey(MAX_LINE_KEY)) {
maxLineResults = Integer.valueOf(basePropretiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY));
}
}
private SqlCompleter createOrUpdateSqlCompleter(SqlCompleter sqlCompleter,
final Connection connection, String propertyKey, final String buf, final int cursor) {
private SqlCompleter createOrUpdateSqlCompleter(
SqlCompleter sqlCompleter,
final Connection connection,
String propertyKey,
final String buf,
final int cursor) {
String schemaFiltersKey = String.format("%s.%s", propertyKey, COMPLETER_SCHEMA_FILTERS_KEY);
String sqlCompleterTtlKey = String.format("%s.%s", propertyKey, COMPLETER_TTL_KEY);
final String schemaFiltersString = getProperty(schemaFiltersKey);
int ttlInSeconds = Integer.valueOf(
StringUtils.defaultIfEmpty(getProperty(sqlCompleterTtlKey), DEFAULT_COMPLETER_TTL)
);
int ttlInSeconds =
Integer.valueOf(
StringUtils.defaultIfEmpty(getProperty(sqlCompleterTtlKey), DEFAULT_COMPLETER_TTL));
final SqlCompleter completer;
if (sqlCompleter == null) {
completer = new SqlCompleter(ttlInSeconds);
@ -243,12 +244,13 @@ public class JDBCInterpreter extends KerberosInterpreter {
completer = sqlCompleter;
}
ExecutorService executorService = Executors.newFixedThreadPool(1);
executorService.execute(new Runnable() {
@Override
public void run() {
completer.createOrUpdateFromConnection(connection, schemaFiltersString, buf, cursor);
}
});
executorService.execute(
new Runnable() {
@Override
public void run() {
completer.createOrUpdateFromConnection(connection, schemaFiltersString, buf, cursor);
}
});
executorService.shutdown();
@ -322,13 +324,13 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
private boolean existAccountInBaseProperty(String propertyKey) {
return basePropretiesMap.get(propertyKey).containsKey(USER_KEY) &&
!isEmpty((String) basePropretiesMap.get(propertyKey).get(USER_KEY)) &&
basePropretiesMap.get(propertyKey).containsKey(PASSWORD_KEY);
return basePropretiesMap.get(propertyKey).containsKey(USER_KEY)
&& !isEmpty((String) basePropretiesMap.get(propertyKey).get(USER_KEY))
&& basePropretiesMap.get(propertyKey).containsKey(PASSWORD_KEY);
}
private UsernamePassword getUsernamePassword(InterpreterContext interpreterContext,
String replName) {
private UsernamePassword getUsernamePassword(
InterpreterContext interpreterContext, String replName) {
UserCredentials uc = interpreterContext.getAuthenticationInfo().getUserCredentials();
if (uc != null) {
return uc.getUsernamePassword(replName);
@ -360,8 +362,8 @@ public class JDBCInterpreter extends KerberosInterpreter {
String user = interpreterContext.getAuthenticationInfo().getUser();
JDBCUserConfigurations jdbcUserConfigurations = getJDBCConfiguration(user);
if (basePropretiesMap.get(propertyKey).containsKey(USER_KEY) &&
!basePropretiesMap.get(propertyKey).getProperty(USER_KEY).isEmpty()) {
if (basePropretiesMap.get(propertyKey).containsKey(USER_KEY)
&& !basePropretiesMap.get(propertyKey).getProperty(USER_KEY).isEmpty()) {
String password = getPassword(basePropretiesMap.get(propertyKey));
if (!isEmpty(password)) {
basePropretiesMap.get(propertyKey).setProperty(PASSWORD_KEY, password);
@ -373,8 +375,8 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
jdbcUserConfigurations.cleanUserProperty(propertyKey);
UsernamePassword usernamePassword = getUsernamePassword(interpreterContext,
getEntityName(interpreterContext.getReplName()));
UsernamePassword usernamePassword =
getUsernamePassword(interpreterContext, getEntityName(interpreterContext.getReplName()));
if (usernamePassword != null) {
jdbcUserConfigurations.setUserProperty(propertyKey, usernamePassword);
} else {
@ -382,13 +384,13 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
}
private void createConnectionPool(String url, String user, String propertyKey,
Properties properties) throws SQLException, ClassNotFoundException {
ConnectionFactory connectionFactory =
new DriverManagerConnectionFactory(url, properties);
private void createConnectionPool(
String url, String user, String propertyKey, Properties properties)
throws SQLException, ClassNotFoundException {
ConnectionFactory connectionFactory = new DriverManagerConnectionFactory(url, properties);
PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory(
connectionFactory, null);
PoolableConnectionFactory poolableConnectionFactory =
new PoolableConnectionFactory(connectionFactory, null);
final String maxConnectionLifetime =
StringUtils.defaultIfEmpty(getProperty("zeppelin.jdbc.maxConnLifetime"), "-1");
poolableConnectionFactory.setMaxConnLifetimeMillis(Long.parseLong(maxConnectionLifetime));
@ -402,8 +404,9 @@ public class JDBCInterpreter extends KerberosInterpreter {
getJDBCConfiguration(user).saveDBDriverPool(propertyKey, driver);
}
private Connection getConnectionFromPool(String url, String user, String propertyKey,
Properties properties) throws SQLException, ClassNotFoundException {
private Connection getConnectionFromPool(
String url, String user, String propertyKey, Properties properties)
throws SQLException, ClassNotFoundException {
String jdbcDriver = getJDBCDriverName(user, propertyKey);
if (!getJDBCConfiguration(user).isConnectionInDBDriverPool(propertyKey)) {
@ -414,7 +417,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
public Connection getConnection(String propertyKey, InterpreterContext interpreterContext)
throws ClassNotFoundException, SQLException, InterpreterException, IOException {
final String user = interpreterContext.getAuthenticationInfo().getUser();
final String user = interpreterContext.getAuthenticationInfo().getUser();
Connection connection;
if (propertyKey == null || basePropretiesMap.get(propertyKey) == null) {
return null;
@ -437,8 +440,9 @@ public class JDBCInterpreter extends KerberosInterpreter {
JDBCSecurityImpl.createSecureConfiguration(getProperties(), authType);
switch (authType) {
case KERBEROS:
if (user == null || "false".equalsIgnoreCase(
getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) {
if (user == null
|| "false"
.equalsIgnoreCase(getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) {
connection = getConnectionFromPool(connectionUrl, user, propertyKey, properties);
} else {
if (basePropretiesMap.get(propertyKey).containsKey("proxy.user.property")) {
@ -446,8 +450,9 @@ public class JDBCInterpreter extends KerberosInterpreter {
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(
user, UserGroupInformation.getCurrentUser());
ugi =
UserGroupInformation.createProxyUser(
user, UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in getCurrentUser", e);
throw new InterpreterException("Error in getCurrentUser", e);
@ -455,12 +460,14 @@ public class JDBCInterpreter extends KerberosInterpreter {
final String poolKey = propertyKey;
try {
connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return getConnectionFromPool(connectionUrl, user, poolKey, properties);
}
});
connection =
ugi.doAs(
new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return getConnectionFromPool(connectionUrl, user, poolKey, properties);
}
});
} catch (Exception e) {
logger.error("Error in doAs", e);
throw new InterpreterException("Error in doAs", e);
@ -480,21 +487,29 @@ public class JDBCInterpreter extends KerberosInterpreter {
private String appendProxyUserToURL(String url, String user, String propertyKey) {
StringBuilder connectionUrl = new StringBuilder(url);
if (user != null && !user.equals("anonymous") &&
basePropretiesMap.get(propertyKey).containsKey("proxy.user.property")) {
if (user != null
&& !user.equals("anonymous")
&& basePropretiesMap.get(propertyKey).containsKey("proxy.user.property")) {
Integer lastIndexOfUrl = connectionUrl.indexOf("?");
if (lastIndexOfUrl == -1) {
lastIndexOfUrl = connectionUrl.length();
}
logger.info("Using proxy user as :" + user);
logger.info("Using proxy property for user as :" +
basePropretiesMap.get(propertyKey).getProperty("proxy.user.property"));
connectionUrl.insert(lastIndexOfUrl, ";" +
basePropretiesMap.get(propertyKey).getProperty("proxy.user.property") + "=" + user + ";");
logger.info(
"Using proxy property for user as :"
+ basePropretiesMap.get(propertyKey).getProperty("proxy.user.property"));
connectionUrl.insert(
lastIndexOfUrl,
";"
+ basePropretiesMap.get(propertyKey).getProperty("proxy.user.property")
+ "="
+ user
+ ";");
} else if (user != null && !user.equals("anonymous") && url.contains("hive")) {
logger.warn("User impersonation for hive has changed please refer: http://zeppelin.apache" +
".org/docs/latest/interpreter/jdbc.html#apache-hive");
logger.warn(
"User impersonation for hive has changed please refer: http://zeppelin.apache"
+ ".org/docs/latest/interpreter/jdbc.html#apache-hive");
}
return connectionUrl.toString();
@ -507,7 +522,8 @@ public class JDBCInterpreter extends KerberosInterpreter {
&& isNotEmpty(properties.getProperty(JDBC_JCEKS_CREDENTIAL_KEY))) {
try {
Configuration configuration = new Configuration();
configuration.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
configuration.set(
CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
properties.getProperty(JDBC_JCEKS_FILE));
CredentialProvider provider = CredentialProviderFactory.getProviders(configuration).get(0);
CredentialProvider.CredentialEntry credEntry =
@ -515,13 +531,18 @@ public class JDBCInterpreter extends KerberosInterpreter {
if (credEntry != null) {
return new String(credEntry.getCredential());
} else {
throw new InterpreterException("Failed to retrieve password from JCEKS from key: "
+ properties.getProperty(JDBC_JCEKS_CREDENTIAL_KEY));
throw new InterpreterException(
"Failed to retrieve password from JCEKS from key: "
+ properties.getProperty(JDBC_JCEKS_CREDENTIAL_KEY));
}
} catch (Exception e) {
logger.error("Failed to retrieve password from JCEKS \n" +
"For file: " + properties.getProperty(JDBC_JCEKS_FILE) +
"\nFor key: " + properties.getProperty(JDBC_JCEKS_CREDENTIAL_KEY), e);
logger.error(
"Failed to retrieve password from JCEKS \n"
+ "For file: "
+ properties.getProperty(JDBC_JCEKS_FILE)
+ "\nFor key: "
+ properties.getProperty(JDBC_JCEKS_CREDENTIAL_KEY),
e);
throw e;
}
}
@ -617,7 +638,10 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
}
if (!quoteString && !doubleQuoteString && !multiLineComment && !singleLineComment
if (!quoteString
&& !doubleQuoteString
&& !multiLineComment
&& !singleLineComment
&& sql.length() > item + 1) {
if (character == '-' && sql.charAt(item + 1) == '-') {
singleLineComment = true;
@ -626,7 +650,10 @@ public class JDBCInterpreter extends KerberosInterpreter {
}
}
if (character == ';' && !quoteString && !doubleQuoteString && !multiLineComment
if (character == ';'
&& !quoteString
&& !doubleQuoteString
&& !multiLineComment
&& !singleLineComment) {
queries.add(StringUtils.trim(query.toString()));
query = new StringBuilder();
@ -656,8 +683,8 @@ public class JDBCInterpreter extends KerberosInterpreter {
return interpreterResult;
}
private InterpreterResult executeSql(String propertyKey, String sql,
InterpreterContext interpreterContext) {
private InterpreterResult executeSql(
String propertyKey, String sql, InterpreterContext interpreterContext) {
Connection connection = null;
Statement statement;
ResultSet resultSet = null;
@ -712,7 +739,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
String statementPrecode =
getProperty(String.format(STATEMENT_PRECODE_KEY_TEMPLATE, propertyKey));
if (StringUtils.isNotBlank(statementPrecode)) {
statement.execute(statementPrecode);
}
@ -723,37 +750,42 @@ public class JDBCInterpreter extends KerberosInterpreter {
resultSet = statement.getResultSet();
// Regards that the command is DDL.
if (isDDLCommand(statement.getUpdateCount(),
resultSet.getMetaData().getColumnCount())) {
interpreterResult.add(InterpreterResult.Type.TEXT,
"Query executed successfully.");
if (isDDLCommand(
statement.getUpdateCount(), resultSet.getMetaData().getColumnCount())) {
interpreterResult.add(InterpreterResult.Type.TEXT, "Query executed successfully.");
} else {
MutableBoolean isComplete = new MutableBoolean(true);
String results = getResults(resultSet,
!containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE), isComplete);
String results =
getResults(
resultSet, !containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE), isComplete);
interpreterResult.add(results);
if (!isComplete.booleanValue()) {
interpreterResult.add(ResultMessages.getExceedsLimitRowsMessage(getMaxResult(),
String.format("%s.%s", COMMON_KEY, MAX_LINE_KEY)));
interpreterResult.add(
ResultMessages.getExceedsLimitRowsMessage(
getMaxResult(), String.format("%s.%s", COMMON_KEY, MAX_LINE_KEY)));
}
}
} else {
// Response contains either an update count or there are no results.
int updateCount = statement.getUpdateCount();
interpreterResult.add(InterpreterResult.Type.TEXT,
"Query executed successfully. Affected rows : " +
updateCount);
interpreterResult.add(
InterpreterResult.Type.TEXT,
"Query executed successfully. Affected rows : " + updateCount);
}
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) { /*ignored*/ }
} catch (SQLException e) {
/*ignored*/
}
}
if (statement != null) {
try {
statement.close();
} catch (SQLException e) { /*ignored*/ }
} catch (SQLException e) {
/*ignored*/
}
}
}
}
@ -763,23 +795,23 @@ public class JDBCInterpreter extends KerberosInterpreter {
interpreterResult.add(errorMsg);
return new InterpreterResult(Code.ERROR, interpreterResult.message());
} finally {
//In case user ran an insert/update/upsert statement
// In case user ran an insert/update/upsert statement
if (connection != null) {
try {
if (!connection.getAutoCommit()) {
connection.commit();
}
connection.close();
} catch (SQLException e) { /*ignored*/ }
} catch (SQLException e) {
/*ignored*/
}
}
getJDBCConfiguration(user).removeStatement(paragraphId);
}
return interpreterResult;
}
/**
* For %table response replace Tab and Newline characters from the content.
*/
/** For %table response replace Tab and Newline characters from the content. */
private String replaceReservedChars(String str) {
if (str == null) {
return EMPTY_COLUMN_VALUE;
@ -789,8 +821,10 @@ public class JDBCInterpreter extends KerberosInterpreter {
@Override
public InterpreterResult interpret(String originalCmd, InterpreterContext contextInterpreter) {
String cmd = Boolean.parseBoolean(getProperty("zeppelin.jdbc.interpolation")) ?
interpolate(originalCmd, contextInterpreter.getResourcePool()) : originalCmd;
String cmd =
Boolean.parseBoolean(getProperty("zeppelin.jdbc.interpolation"))
? interpolate(originalCmd, contextInterpreter.getResourcePool())
: originalCmd;
logger.debug("Run SQL command '{}'", cmd);
String propertyKey = getPropertyKey(contextInterpreter);
cmd = cmd.trim();
@ -803,7 +837,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
logger.info("Cancel current query statement.");
String paragraphId = context.getParagraphId();
JDBCUserConfigurations jdbcUserConfigurations =
getJDBCConfiguration(context.getAuthenticationInfo().getUser());
getJDBCConfiguration(context.getAuthenticationInfo().getUser());
try {
jdbcUserConfigurations.cancelStatement(paragraphId);
} catch (SQLException e) {
@ -839,15 +873,15 @@ public class JDBCInterpreter extends KerberosInterpreter {
@Override
public Scheduler getScheduler() {
String schedulerName = JDBCInterpreter.class.getName() + this.hashCode();
return isConcurrentExecution() ?
SchedulerFactory.singleton().createOrGetParallelScheduler(schedulerName,
getMaxConcurrentConnection())
: SchedulerFactory.singleton().createOrGetFIFOScheduler(schedulerName);
return isConcurrentExecution()
? SchedulerFactory.singleton()
.createOrGetParallelScheduler(schedulerName, getMaxConcurrentConnection())
: SchedulerFactory.singleton().createOrGetFIFOScheduler(schedulerName);
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) throws InterpreterException {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) throws InterpreterException {
List<InterpreterCompletion> candidates = new ArrayList<>();
String propertyKey = getPropertyKey(interpreterContext);
String sqlCompleterKey =

19
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCUserConfigurations.java
View file

@ -5,28 +5,24 @@
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jdbc;
import org.apache.commons.dbcp2.PoolingDriver;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.dbcp2.PoolingDriver;
import org.apache.zeppelin.user.UsernamePassword;
/**
* UserConfigurations for JDBC impersonation.
*/
/** UserConfigurations for JDBC impersonation. */
public class JDBCUserConfigurations {
private final Map<String, Statement> paragraphIdStatementMap;
private final Map<String, PoolingDriver> poolingDriverMap;
@ -87,6 +83,7 @@ public class JDBCUserConfigurations {
poolingDriverMap.put(key, driver);
isSuccessful.put(key, false);
}
public PoolingDriver removeDBDriverPool(String key) throws SQLException {
isSuccessful.remove(key);
return poolingDriverMap.remove(key);

218
jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
View file

@ -4,11 +4,6 @@ package org.apache.zeppelin.jdbc;
* This source file is based on code taken from SQLLine 1.0.2 See SQLLine notice in LICENSE
*/
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@ -26,54 +21,47 @@ import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeSet;
import java.util.regex.Pattern;
import jline.console.completer.ArgumentCompleter.ArgumentList;
import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.zeppelin.completer.CachedCompleter;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.completer.StringsCompleter;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* SQL auto complete functionality for the JdbcInterpreter.
*/
/** SQL auto complete functionality for the JdbcInterpreter. */
public class SqlCompleter {
private static Logger logger = LoggerFactory.getLogger(SqlCompleter.class);
/**
* Delimiter that can split SQL statement in keyword list.
*/
private WhitespaceArgumentDelimiter sqlDelimiter = new WhitespaceArgumentDelimiter() {
/** Delimiter that can split SQL statement in keyword list. */
private WhitespaceArgumentDelimiter sqlDelimiter =
new WhitespaceArgumentDelimiter() {
private Pattern pattern = Pattern.compile(",");
private Pattern pattern = Pattern.compile(",");
@Override
public boolean isDelimiterChar(CharSequence buffer, int pos) {
return pattern.matcher("" + buffer.charAt(pos)).matches()
@Override
public boolean isDelimiterChar(CharSequence buffer, int pos) {
return pattern.matcher("" + buffer.charAt(pos)).matches()
|| super.isDelimiterChar(buffer, pos);
}
};
}
};
/**
* Schema completer.
*/
/** Schema completer. */
private CachedCompleter schemasCompleter;
/**
* Contain different completer with table list for every schema name.
*/
/** Contain different completer with table list for every schema name. */
private Map<String, CachedCompleter> tablesCompleters = new HashMap<>();
/**
* Contains different completer with column list for every table name
* Table names store as schema_name.table_name.
* Contains different completer with column list for every table name Table names store as
* schema_name.table_name.
*/
private Map<String, CachedCompleter> columnsCompleters = new HashMap<>();
/**
* Completer for sql keywords.
*/
/** Completer for sql keywords. */
private CachedCompleter keywordCompleter;
private int ttlInSeconds;
@ -99,7 +87,11 @@ public class SqlCompleter {
argumentPosition = argumentList.getArgumentPosition();
}
int complete = completeName(cursorArgument, argumentPosition, candidates,
int complete =
completeName(
cursorArgument,
argumentPosition,
candidates,
findAliasesInSQL(argumentList.getArguments()));
logger.debug("complete:" + complete + ", size:" + candidates.size());
@ -110,10 +102,9 @@ public class SqlCompleter {
* Return list of schema names within the database.
*
* @param meta metadata from connection to database
* @param schemaFilters a schema name patterns; must match the schema name
* as it is stored in the database; "" retrieves those without a schema;
* <code>null</code> means that the schema name should not be used to narrow
* the search; supports '%'; for example "prod_v_%"
* @param schemaFilters a schema name patterns; must match the schema name as it is stored in the
* database; "" retrieves those without a schema; <code>null</code> means that the schema name
* should not be used to narrow the search; supports '%'; for example "prod_v_%"
* @return set of all schema names in the database
*/
private static Set<String> getSchemaNames(DatabaseMetaData meta, List<String> schemaFilters) {
@ -146,10 +137,9 @@ public class SqlCompleter {
* Return list of catalog names within the database.
*
* @param meta metadata from connection to database
* @param schemaFilters a catalog name patterns; must match the catalog name
* as it is stored in the database; "" retrieves those without a catalog;
* <code>null</code> means that the schema name should not be used to narrow
* the search; supports '%'; for example "prod_v_%"
* @param schemaFilters a catalog name patterns; must match the catalog name as it is stored in
* the database; "" retrieves those without a catalog; <code>null</code> means that the schema
* name should not be used to narrow the search; supports '%'; for example "prod_v_%"
* @return set of all catalog names in the database
*/
private static Set<String> getCatalogNames(DatabaseMetaData meta, List<String> schemaFilters) {
@ -175,8 +165,14 @@ public class SqlCompleter {
}
private static void fillTableNames(String schema, DatabaseMetaData meta, Set<String> tables) {
try (ResultSet tbls = meta.getTables(schema, schema, "%",
new String[]{"TABLE", "VIEW", "ALIAS", "SYNONYM", "GLOBAL TEMPORARY", "LOCAL TEMPORARY"})) {
try (ResultSet tbls =
meta.getTables(
schema,
schema,
"%",
new String[] {
"TABLE", "VIEW", "ALIAS", "SYNONYM", "GLOBAL TEMPORARY", "LOCAL TEMPORARY"
})) {
while (tbls.next()) {
String table = tbls.getString("TABLE_NAME");
tables.add(table);
@ -192,11 +188,11 @@ public class SqlCompleter {
* @param schema name of a scheme
* @param table name of a table
* @param meta meta metadata from connection to database
* @param columns function fills this set, for every table name adds set
* of columns within the table; table name is in format schema_name.table_name
* @param columns function fills this set, for every table name adds set of columns within the
* table; table name is in format schema_name.table_name
*/
private static void fillColumnNames(String schema, String table, DatabaseMetaData meta,
Set<String> columns) {
private static void fillColumnNames(
String schema, String table, DatabaseMetaData meta, Set<String> columns) {
try (ResultSet cols = meta.getColumns(schema, schema, table, "%")) {
while (cols.next()) {
String column = cols.getString("COLUMN_NAME");
@ -207,31 +203,34 @@ public class SqlCompleter {
}
}
public static Set<String> getSqlKeywordsCompletions(DatabaseMetaData meta) throws IOException,
SQLException {
public static Set<String> getSqlKeywordsCompletions(DatabaseMetaData meta)
throws IOException, SQLException {
// Add the default SQL completions
String keywords =
new BufferedReader(new InputStreamReader(
SqlCompleter.class.getResourceAsStream("/ansi.sql.keywords"))).readLine();
new BufferedReader(
new InputStreamReader(SqlCompleter.class.getResourceAsStream("/ansi.sql.keywords")))
.readLine();
Set<String> completions = new TreeSet<>();
if (null != meta) {
// Add the driver specific SQL completions
String driverSpecificKeywords =
"/" + meta.getDriverName().replace(" ", "-").toLowerCase() + "-sql.keywords";
"/" + meta.getDriverName().replace(" ", "-").toLowerCase() + "-sql.keywords";
logger.info("JDBC DriverName:" + driverSpecificKeywords);
try {
if (SqlCompleter.class.getResource(driverSpecificKeywords) != null) {
String driverKeywords =
new BufferedReader(new InputStreamReader(
new BufferedReader(
new InputStreamReader(
SqlCompleter.class.getResourceAsStream(driverSpecificKeywords)))
.readLine();
.readLine();
keywords += "," + driverKeywords.toUpperCase();
}
} catch (Exception e) {
logger.debug("fail to get driver specific SQL completions for " +
driverSpecificKeywords + " : " + e, e);
logger.debug(
"fail to get driver specific SQL completions for " + driverSpecificKeywords + " : " + e,
e);
}
// Add the keywords from the current JDBC connection
@ -277,11 +276,11 @@ public class SqlCompleter {
* Initializes all local completers from database connection.
*
* @param connection database connection
* @param schemaFiltersString a comma separated schema name patterns, supports '%' symbol;
* for example "prod_v_%,prod_t_%"
* @param schemaFiltersString a comma separated schema name patterns, supports '%' symbol; for
* example "prod_v_%,prod_t_%"
*/
public void createOrUpdateFromConnection(Connection connection, String schemaFiltersString,
String buffer, int cursor) {
public void createOrUpdateFromConnection(
Connection connection, String schemaFiltersString, String buffer, int cursor) {
try (Connection c = connection) {
if (schemaFiltersString == null) {
schemaFiltersString = StringUtils.EMPTY;
@ -297,14 +296,16 @@ public class SqlCompleter {
if (c != null) {
DatabaseMetaData databaseMetaData = c.getMetaData();
if (keywordCompleter == null || keywordCompleter.getCompleter() == null
if (keywordCompleter == null
|| keywordCompleter.getCompleter() == null
|| keywordCompleter.isExpired()) {
keywords = getSqlKeywordsCompletions(databaseMetaData);
initKeywords(keywords);
}
if (cursorArgument.needLoadSchemas() &&
(schemasCompleter == null || schemasCompleter.getCompleter() == null
|| schemasCompleter.isExpired())) {
if (cursorArgument.needLoadSchemas()
&& (schemasCompleter == null
|| schemasCompleter.getCompleter() == null
|| schemasCompleter.isExpired())) {
schemas = getSchemaNames(databaseMetaData, schemaFilters);
catalogs = getCatalogNames(databaseMetaData, schemaFilters);
@ -316,8 +317,8 @@ public class SqlCompleter {
}
CachedCompleter tablesCompleter = tablesCompleters.get(cursorArgument.getSchema());
if (cursorArgument.needLoadTables() &&
(tablesCompleter == null || tablesCompleter.isExpired())) {
if (cursorArgument.needLoadTables()
&& (tablesCompleter == null || tablesCompleter.isExpired())) {
fillTableNames(cursorArgument.getSchema(), databaseMetaData, tables);
initTables(cursorArgument.getSchema(), tables);
}
@ -326,15 +327,21 @@ public class SqlCompleter {
String.format("%s.%s", cursorArgument.getSchema(), cursorArgument.getTable());
CachedCompleter columnsCompleter = columnsCompleters.get(schemaTable);
if (cursorArgument.needLoadColumns() &&
(columnsCompleter == null || columnsCompleter.isExpired())) {
fillColumnNames(cursorArgument.getSchema(), cursorArgument.getTable(), databaseMetaData,
columns);
if (cursorArgument.needLoadColumns()
&& (columnsCompleter == null || columnsCompleter.isExpired())) {
fillColumnNames(
cursorArgument.getSchema(), cursorArgument.getTable(), databaseMetaData, columns);
initColumns(schemaTable, columns);
}
logger.info("Completer initialized with " + schemas.size() + " schemas, " +
columns.size() + " tables and " + keywords.size() + " keywords");
logger.info(
"Completer initialized with "
+ schemas.size()
+ " schemas, "
+ columns.size()
+ " tables and "
+ keywords.size()
+ " keywords");
}
} catch (SQLException | IOException e) {
@ -350,22 +357,22 @@ public class SqlCompleter {
public void initSchemas(Set<String> schemas) {
if (schemas != null && !schemas.isEmpty()) {
schemasCompleter = new CachedCompleter(
new StringsCompleter(new TreeSet<>(schemas)), ttlInSeconds);
schemasCompleter =
new CachedCompleter(new StringsCompleter(new TreeSet<>(schemas)), ttlInSeconds);
}
}
public void initTables(String schema, Set<String> tables) {
if (tables != null && !tables.isEmpty()) {
tablesCompleters.put(schema, new CachedCompleter(
new StringsCompleter(new TreeSet<>(tables)), ttlInSeconds));
tablesCompleters.put(
schema, new CachedCompleter(new StringsCompleter(new TreeSet<>(tables)), ttlInSeconds));
}
}
public void initColumns(String schemaTable, Set<String> columns) {
if (columns != null && !columns.isEmpty()) {
columnsCompleters.put(schemaTable,
new CachedCompleter(new StringsCompleter(columns), ttlInSeconds));
columnsCompleters.put(
schemaTable, new CachedCompleter(new StringsCompleter(columns), ttlInSeconds));
}
}
@ -378,8 +385,8 @@ public class SqlCompleter {
public Map<String, String> findAliasesInSQL(String[] sqlArguments) {
Map<String, String> res = new HashMap<>();
for (int i = 0; i < sqlArguments.length - 1; i++) {
if (columnsCompleters.keySet().contains(sqlArguments[i]) &&
sqlArguments[i + 1].matches("[a-zA-Z]+")) {
if (columnsCompleters.keySet().contains(sqlArguments[i])
&& sqlArguments[i + 1].matches("[a-zA-Z]+")) {
res.put(sqlArguments[i + 1], sqlArguments[i]);
}
}
@ -409,8 +416,8 @@ public class SqlCompleter {
*
* @return -1 in case of no candidates found, 0 otherwise
*/
private int completeTable(String schema, String buffer, int cursor,
List<CharSequence> candidates) {
private int completeTable(
String schema, String buffer, int cursor, List<CharSequence> candidates) {
// Wrong schema
if (schema == null || !tablesCompleters.containsKey(schema)) {
return -1;
@ -424,26 +431,31 @@ public class SqlCompleter {
*
* @return -1 in case of no candidates found, 0 otherwise
*/
private int completeColumn(String schema, String table, String buffer, int cursor,
List<CharSequence> candidates) {
private int completeColumn(
String schema, String table, String buffer, int cursor, List<CharSequence> candidates) {
// Wrong schema or wrong table
if (schema == null || table == null || !columnsCompleters.containsKey(schema + "." + table)) {
return -1;
} else {
return columnsCompleters.get(schema + "." + table).getCompleter()
return columnsCompleters
.get(schema + "." + table)
.getCompleter()
.complete(buffer, cursor, candidates);
}
}
/**
* Complete buffer with a single name. Function will decide what it is:
* a schema, a table of a column or a keyword
* Complete buffer with a single name. Function will decide what it is: a schema, a table of a
* column or a keyword
*
* @param aliases for every alias contains table name in format schema_name.table_name
* @return -1 in case of no candidates found, 0 otherwise
*/
public int completeName(String buffer, int cursor, List<InterpreterCompletion> candidates,
Map<String, String> aliases) {
public int completeName(
String buffer,
int cursor,
List<InterpreterCompletion> candidates,
Map<String, String> aliases) {
CursorArgument cursorArgument = parseCursorArgument(buffer, cursor);
// find schema and table name if they are
@ -451,40 +463,42 @@ public class SqlCompleter {
String table;
String column;
if (cursorArgument.getSchema() == null) { // process all
if (cursorArgument.getSchema() == null) { // process all
List<CharSequence> keywordsCandidates = new ArrayList();
List<CharSequence> schemaCandidates = new ArrayList<>();
int keywordsRes = completeKeyword(buffer, cursor, keywordsCandidates);
int schemaRes = completeSchema(buffer, cursor, schemaCandidates);
addCompletions(candidates, keywordsCandidates, CompletionType.keyword.name());
addCompletions(candidates, schemaCandidates, CompletionType.schema.name());
return NumberUtils.max(new int[]{keywordsRes, schemaRes});
return NumberUtils.max(new int[] {keywordsRes, schemaRes});
} else {
schema = cursorArgument.getSchema();
if (aliases.containsKey(schema)) { // process alias case
if (aliases.containsKey(schema)) { // process alias case
String alias = aliases.get(schema);
int pointPos = alias.indexOf('.');
schema = alias.substring(0, pointPos);
table = alias.substring(pointPos + 1);
column = cursorArgument.getColumn();
List<CharSequence> columnCandidates = new ArrayList();
int columnRes = completeColumn(schema, table, column, cursorArgument.getCursorPosition(),
columnCandidates);
int columnRes =
completeColumn(
schema, table, column, cursorArgument.getCursorPosition(), columnCandidates);
addCompletions(candidates, columnCandidates, CompletionType.column.name());
// process schema.table case
} else if (cursorArgument.getTable() != null && cursorArgument.getColumn() == null) {
List<CharSequence> tableCandidates = new ArrayList();
table = cursorArgument.getTable();
int tableRes = completeTable(schema, table, cursorArgument.getCursorPosition(),
tableCandidates);
int tableRes =
completeTable(schema, table, cursorArgument.getCursorPosition(), tableCandidates);
addCompletions(candidates, tableCandidates, CompletionType.table.name());
return tableRes;
} else {
List<CharSequence> columnCandidates = new ArrayList();
table = cursorArgument.getTable();
column = cursorArgument.getColumn();
int columnRes = completeColumn(schema, table, column, cursorArgument.getCursorPosition(),
columnCandidates);
int columnRes =
completeColumn(
schema, table, column, cursorArgument.getCursorPosition(), columnCandidates);
addCompletions(candidates, columnCandidates, CompletionType.column.name());
}
}
@ -497,11 +511,13 @@ public class SqlCompleter {
return this.sqlDelimiter;
}
private void addCompletions(List<InterpreterCompletion> interpreterCompletions,
List<CharSequence> candidates, String meta) {
private void addCompletions(
List<InterpreterCompletion> interpreterCompletions,
List<CharSequence> candidates,
String meta) {
for (CharSequence candidate : candidates) {
interpreterCompletions.add(new InterpreterCompletion(candidate.toString(),
candidate.toString(), meta));
interpreterCompletions.add(
new InterpreterCompletion(candidate.toString(), candidate.toString(), meta));
}
}

40
jdbc/src/main/java/org/apache/zeppelin/jdbc/security/JDBCSecurityImpl.java
View file

@ -19,31 +19,29 @@ package org.apache.zeppelin.jdbc.security;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.SIMPLE;
import java.io.IOException;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Properties;
/**
* Created for org.apache.zeppelin.jdbc.security on 09/07/16.
*/
/** Created for org.apache.zeppelin.jdbc.security on 09/07/16. */
public class JDBCSecurityImpl {
private static final Logger LOGGER = LoggerFactory.getLogger(JDBCSecurityImpl.class);
/***
/**
* *
*
* @param properties
*/
public static void createSecureConfiguration(Properties properties,
AuthenticationMethod authType) {
public static void createSecureConfiguration(
Properties properties, AuthenticationMethod authType) {
switch (authType) {
case KERBEROS:
Configuration conf = new
org.apache.hadoop.conf.Configuration();
Configuration conf = new org.apache.hadoop.conf.Configuration();
conf.set("hadoop.security.authentication", KERBEROS.toString());
UserGroupInformation.setConfiguration(conf);
try {
@ -57,12 +55,13 @@ public class JDBCSecurityImpl {
properties.getProperty("zeppelin.jdbc.principal"),
properties.getProperty("zeppelin.jdbc.keytab.location"));
} else {
LOGGER.info("The user has already logged in using Keytab and principal, " +
"no action required");
LOGGER.info(
"The user has already logged in using Keytab and principal, "
+ "no action required");
}
} catch (IOException e) {
LOGGER.error("Failed to get either keytab location or principal name in the " +
"interpreter", e);
LOGGER.error(
"Failed to get either keytab location or principal name in the " + "interpreter", e);
}
}
}
@ -70,11 +69,14 @@ public class JDBCSecurityImpl {
public static AuthenticationMethod getAuthtype(Properties properties) {
AuthenticationMethod authType;
try {
authType = AuthenticationMethod.valueOf(properties.getProperty("zeppelin.jdbc.auth.type")
.trim().toUpperCase());
authType =
AuthenticationMethod.valueOf(
properties.getProperty("zeppelin.jdbc.auth.type").trim().toUpperCase());
} catch (Exception e) {
LOGGER.error(String.format("Invalid auth.type detected with value %s, defaulting " +
"auth.type to SIMPLE", properties.getProperty("zeppelin.jdbc.auth.type")));
LOGGER.error(
String.format(
"Invalid auth.type detected with value %s, defaulting " + "auth.type to SIMPLE",
properties.getProperty("zeppelin.jdbc.auth.type")));
authType = SIMPLE;
}
return authType;

72
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterInterpolationTest.java
View file

@ -5,16 +5,26 @@
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jdbc;
import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.resource.LocalResourcePool;
@ -23,20 +33,7 @@ import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.util.Properties;
import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
/**
* JDBC interpreter Z-variable interpolation unit tests.
*/
/** JDBC interpreter Z-variable interpolation unit tests. */
public class JDBCInterpreterInterpolationTest extends BasicJDBCTestCaseAdapter {
private static String jdbcConnection;
@ -58,22 +55,24 @@ public class JDBCInterpreterInterpolationTest extends BasicJDBCTestCaseAdapter {
Connection connection = DriverManager.getConnection(getJdbcConnection());
Statement statement = connection.createStatement();
statement.execute(
"DROP TABLE IF EXISTS test_table; " +
"CREATE TABLE test_table(id varchar(255), name varchar(255));");
"DROP TABLE IF EXISTS test_table; "
+ "CREATE TABLE test_table(id varchar(255), name varchar(255));");
Statement insertStatement = connection.createStatement();
insertStatement.execute("insert into test_table(id, name) values " +
"('pro', 'processor')," +
"('mem', 'memory')," +
"('key', 'keyboard')," +
"('mou', 'mouse');");
insertStatement.execute(
"insert into test_table(id, name) values "
+ "('pro', 'processor'),"
+ "('mem', 'memory'),"
+ "('key', 'keyboard'),"
+ "('mou', 'mouse');");
resourcePool = new LocalResourcePool("JdbcInterpolationTest");
interpreterContext = InterpreterContext.builder()
.setParagraphId("paragraph_1")
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.setResourcePool(resourcePool)
.build();
interpreterContext =
InterpreterContext.builder()
.setParagraphId("paragraph_1")
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.setResourcePool(resourcePool)
.build();
}
@Test
@ -110,8 +109,7 @@ public class JDBCInterpreterInterpolationTest extends BasicJDBCTestCaseAdapter {
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals(1, interpreterResult.message().size());
assertEquals("ID\tNAME\nmem\tmemory\n",
interpreterResult.message().get(0).getData());
assertEquals("ID\tNAME\nmem\tmemory\n", interpreterResult.message().get(0).getData());
}
@Test
@ -149,8 +147,7 @@ public class JDBCInterpreterInterpolationTest extends BasicJDBCTestCaseAdapter {
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals(1, interpreterResult.message().size());
assertEquals("ID\tNAME\nkey\tkeyboard\n",
interpreterResult.message().get(0).getData());
assertEquals("ID\tNAME\nkey\tkeyboard\n", interpreterResult.message().get(0).getData());
}
@Test
@ -177,8 +174,7 @@ public class JDBCInterpreterInterpolationTest extends BasicJDBCTestCaseAdapter {
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals(1, interpreterResult.message().size());
assertEquals("ID\tNAME\nkey\tkeyboard\nmou\tmouse\n",
interpreterResult.message().get(0).getData());
assertEquals(
"ID\tNAME\nkey\tkeyboard\nmou\tmouse\n", interpreterResult.message().get(0).getData());
}
}

231
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -5,36 +5,31 @@
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jdbc;
import static java.lang.String.format;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_DRIVER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PRECODE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_STATEMENT_PRECODE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.PRECODE_KEY_TEMPLATE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.STATEMENT_PRECODE_KEY_TEMPLATE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static java.lang.String.format;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_DRIVER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_STATEMENT_PRECODE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PRECODE;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.PRECODE_KEY_TEMPLATE;
import org.junit.Before;
import org.junit.Test;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.STATEMENT_PRECODE_KEY_TEMPLATE;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -47,9 +42,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
@ -61,10 +53,10 @@ import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.UserCredentials;
import org.apache.zeppelin.user.UsernamePassword;
import org.junit.Before;
import org.junit.Test;
/**
* JDBC interpreter unit tests.
*/
/** JDBC interpreter unit tests. */
public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
static String jdbcConnection;
InterpreterContext interpreterContext;
@ -95,40 +87,36 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
Connection connection = DriverManager.getConnection(getJdbcConnection());
Statement statement = connection.createStatement();
statement.execute(
"DROP TABLE IF EXISTS test_table; " +
"CREATE TABLE test_table(id varchar(255), name varchar(255));");
"DROP TABLE IF EXISTS test_table; "
+ "CREATE TABLE test_table(id varchar(255), name varchar(255));");
PreparedStatement insertStatement = connection.prepareStatement(
PreparedStatement insertStatement =
connection.prepareStatement(
"insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);");
insertStatement.setString(1, null);
insertStatement.execute();
interpreterContext = InterpreterContext.builder()
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.build();
interpreterContext =
InterpreterContext.builder()
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.build();
}
@Test
public void testForParsePropertyKey() {
JDBCInterpreter t = new JDBCInterpreter(new Properties());
Map<String, String> localProperties = new HashMap<>();
InterpreterContext interpreterContext = InterpreterContext.builder()
.setLocalProperties(localProperties)
.build();
InterpreterContext interpreterContext =
InterpreterContext.builder().setLocalProperties(localProperties).build();
assertEquals(JDBCInterpreter.DEFAULT_KEY, t.getPropertyKey(interpreterContext));
localProperties = new HashMap<>();
localProperties.put("db", "mysql");
interpreterContext = InterpreterContext.builder()
.setLocalProperties(localProperties)
.build();
interpreterContext = InterpreterContext.builder().setLocalProperties(localProperties).build();
assertEquals("mysql", t.getPropertyKey(interpreterContext));
localProperties = new HashMap<>();
localProperties.put("hive", "hive");
interpreterContext = InterpreterContext.builder()
.setLocalProperties(localProperties)
.build();
interpreterContext = InterpreterContext.builder().setLocalProperties(localProperties).build();
assertEquals("hive", t.getPropertyKey(interpreterContext));
}
@ -147,10 +135,11 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
String sqlQuery = "select * from test_table";
Map<String, String> localProperties = new HashMap<>();
localProperties.put("db", "fake");
InterpreterContext context = InterpreterContext.builder()
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.setLocalProperties(localProperties)
.build();
InterpreterContext context =
InterpreterContext.builder()
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.setLocalProperties(localProperties)
.build();
InterpreterResult interpreterResult = t.interpret(sqlQuery, context);
// if prefix not found return ERROR and Prefix not found.
@ -192,17 +181,17 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
@Test
public void testSplitSqlQuery() throws SQLException, IOException {
String sqlQuery = "insert into test_table(id, name) values ('a', ';\"');" +
"select * from test_table;" +
"select * from test_table WHERE ID = \";'\";" +
"select * from test_table WHERE ID = ';';" +
"select '\n', ';';" +
"select replace('A\\;B', '\\', 'text');" +
"select '\\', ';';" +
"select '''', ';';" +
"select /*+ scan */ * from test_table;" +
"--singleLineComment\nselect * from test_table";
String sqlQuery =
"insert into test_table(id, name) values ('a', ';\"');"
+ "select * from test_table;"
+ "select * from test_table WHERE ID = \";'\";"
+ "select * from test_table WHERE ID = ';';"
+ "select '\n', ';';"
+ "select replace('A\\;B', '\\', 'text');"
+ "select '\\', ';';"
+ "select '''', ';';"
+ "select /*+ scan */ * from test_table;"
+ "--singleLineComment\nselect * from test_table";
Properties properties = new Properties();
JDBCInterpreter t = new JDBCInterpreter(properties);
@ -223,10 +212,11 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
@Test
public void testQueryWithEscapedCharacters() throws SQLException, IOException {
String sqlQuery = "select '\\n', ';';" +
"select replace('A\\;B', '\\', 'text');" +
"select '\\', ';';" +
"select '''', ';'";
String sqlQuery =
"select '\\n', ';';"
+ "select replace('A\\;B', '\\', 'text');"
+ "select '\\', ';';"
+ "select '''', ';'";
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
@ -265,15 +255,14 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table;" +
"select * from test_table WHERE ID = ';';";
String sqlQuery = "select * from test_table;" + "select * from test_table WHERE ID = ';';";
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(2, interpreterResult.message().size());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n",
interpreterResult.message().get(0).getData());
assertEquals(
"ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", interpreterResult.message().get(0).getData());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(1).getType());
assertEquals("ID\tNAME\n", interpreterResult.message().get(1).getData());
@ -291,15 +280,14 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "select * from test_table;" +
"select * from test_table WHERE ID = ';';";
String sqlQuery = "select * from test_table;" + "select * from test_table WHERE ID = ';';";
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
assertEquals(1, interpreterResult.message().size());
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n",
interpreterResult.message().get(0).getData());
assertEquals(
"ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", interpreterResult.message().get(0).getData());
}
@Test
@ -323,7 +311,6 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
assertEquals("ID\tNAME\nc\tnull\n", interpreterResult.message().get(0).getData());
}
@Test
public void testSelectQueryMaxResult() throws SQLException, IOException {
Properties properties = new Properties();
@ -384,11 +371,11 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
jdbcInterpreter.interpret("", interpreterContext);
List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 3,
interpreterContext);
List<InterpreterCompletion> completionList =
jdbcInterpreter.completion("sel", 3, interpreterContext);
InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select", "select",
CompletionType.keyword.name());
InterpreterCompletion correctCompletionKeyword =
new InterpreterCompletion("select", "select", CompletionType.keyword.name());
assertEquals(1, completionList.size());
assertEquals(true, completionList.contains(correctCompletionKeyword));
@ -409,8 +396,8 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
return properties;
}
private AuthenticationInfo getUserAuth(String user, String entityName, String dbUser,
String dbPassword) {
private AuthenticationInfo getUserAuth(
String user, String entityName, String dbUser, String dbPassword) {
UserCredentials userCredentials = new UserCredentials();
if (entityName != null && dbUser != null && dbPassword != null) {
UsernamePassword up = new UsernamePassword(dbUser, dbPassword);
@ -440,10 +427,11 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
// user1 runs jdbc1
jdbc1.open();
InterpreterContext ctx1 = InterpreterContext.builder()
.setAuthenticationInfo(user1Credential)
.setReplName("jdbc1")
.build();
InterpreterContext ctx1 =
InterpreterContext.builder()
.setAuthenticationInfo(user1Credential)
.setReplName("jdbc1")
.build();
jdbc1.interpret("", ctx1);
JDBCUserConfigurations user1JDBC1Conf = jdbc1.getJDBCConfiguration("user1");
@ -453,10 +441,11 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
// user1 runs jdbc2
jdbc2.open();
InterpreterContext ctx2 = InterpreterContext.builder()
.setAuthenticationInfo(user1Credential)
.setReplName("jdbc2")
.build();
InterpreterContext ctx2 =
InterpreterContext.builder()
.setAuthenticationInfo(user1Credential)
.setReplName("jdbc2")
.build();
jdbc2.interpret("", ctx2);
JDBCUserConfigurations user1JDBC2Conf = jdbc2.getJDBCConfiguration("user1");
@ -466,10 +455,11 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
// user2 runs jdbc1
jdbc1.open();
InterpreterContext ctx3 = InterpreterContext.builder()
.setAuthenticationInfo(user2Credential)
.setReplName("jdbc1")
.build();
InterpreterContext ctx3 =
InterpreterContext.builder()
.setAuthenticationInfo(user2Credential)
.setReplName("jdbc1")
.build();
jdbc1.interpret("", ctx3);
JDBCUserConfigurations user2JDBC1Conf = jdbc1.getJDBCConfiguration("user2");
@ -479,10 +469,11 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
// user2 runs jdbc2
jdbc2.open();
InterpreterContext ctx4 = InterpreterContext.builder()
.setAuthenticationInfo(user2Credential)
.setReplName("jdbc2")
.build();
InterpreterContext ctx4 =
InterpreterContext.builder()
.setAuthenticationInfo(user2Credential)
.setReplName("jdbc2")
.build();
jdbc2.interpret("", ctx4);
JDBCUserConfigurations user2JDBC2Conf = jdbc2.getJDBCConfiguration("user2");
@ -498,8 +489,9 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
properties.setProperty("default.url", getJdbcConnection());
properties.setProperty("default.user", "");
properties.setProperty("default.password", "");
properties.setProperty(DEFAULT_PRECODE,
"create table test_precode (id int); insert into test_precode values (1);");
properties.setProperty(
DEFAULT_PRECODE,
"create table test_precode (id int); insert into test_precode values (1);");
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
jdbcInterpreter.open();
jdbcInterpreter.executePrecode(interpreterContext);
@ -541,17 +533,19 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
properties.setProperty("anotherPrefix.url", getJdbcConnection());
properties.setProperty("anotherPrefix.user", "");
properties.setProperty("anotherPrefix.password", "");
properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"),
"create table test_precode_2 (id int); insert into test_precode_2 values (2);");
properties.setProperty(
String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"),
"create table test_precode_2 (id int); insert into test_precode_2 values (2);");
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
jdbcInterpreter.open();
Map<String, String> localProperties = new HashMap<>();
localProperties.put("db", "anotherPrefix");
InterpreterContext context = InterpreterContext.builder()
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.setLocalProperties(localProperties)
.build();
InterpreterContext context =
InterpreterContext.builder()
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.setLocalProperties(localProperties)
.build();
jdbcInterpreter.executePrecode(context);
String sqlQuery = "select * from test_precode_2";
@ -609,17 +603,19 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
properties.setProperty("anotherPrefix.url", getJdbcConnection());
properties.setProperty("anotherPrefix.user", "");
properties.setProperty("anotherPrefix.password", "");
properties.setProperty(String.format(STATEMENT_PRECODE_KEY_TEMPLATE, "anotherPrefix"),
"set @v='statementAnotherPrefix'");
properties.setProperty(
String.format(STATEMENT_PRECODE_KEY_TEMPLATE, "anotherPrefix"),
"set @v='statementAnotherPrefix'");
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
jdbcInterpreter.open();
Map<String, String> localProperties = new HashMap<>();
localProperties.put("db", "anotherPrefix");
InterpreterContext context = InterpreterContext.builder()
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.setLocalProperties(localProperties)
.build();
InterpreterContext context =
InterpreterContext.builder()
.setAuthenticationInfo(new AuthenticationInfo("testUser"))
.setLocalProperties(localProperties)
.build();
String sqlQuery = "select @v";
@ -643,16 +639,17 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
JDBCInterpreter t = new JDBCInterpreter(properties);
t.open();
String sqlQuery = "/* ; */\n" +
"-- /* comment\n" +
"--select * from test_table\n" +
"select * from test_table; /* some comment ; */\n" +
"/*\n" +
"select * from test_table;\n" +
"*/\n" +
"-- a ; b\n" +
"select * from test_table WHERE ID = ';--';\n" +
"select * from test_table WHERE ID = '/*' -- test";
String sqlQuery =
"/* ; */\n"
+ "-- /* comment\n"
+ "--select * from test_table\n"
+ "select * from test_table; /* some comment ; */\n"
+ "/*\n"
+ "select * from test_table;\n"
+ "*/\n"
+ "-- a ; b\n"
+ "select * from test_table WHERE ID = ';--';\n"
+ "select * from test_table WHERE ID = '/*' -- test";
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());

270
jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java
View file

@ -5,29 +5,20 @@
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.jdbc;
import static com.google.common.collect.Sets.newHashSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import com.google.common.base.Joiner;
import org.apache.commons.lang.StringUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
@ -36,15 +27,17 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import jline.console.completer.ArgumentCompleter;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.completer.CompletionType;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* SQL completer unit tests.
*/
/** SQL completer unit tests. */
public class SqlCompleterTest {
public class CompleterTester {
private SqlCompleter completer;
@ -84,8 +77,8 @@ public class SqlCompleterTest {
}
}
private void expectedCompletions(String buffer, int cursor,
Set<InterpreterCompletion> expected) {
private void expectedCompletions(
String buffer, int cursor, Set<InterpreterCompletion> expected) {
if (StringUtils.isNotEmpty(buffer) && buffer.length() > cursor) {
buffer = buffer.substring(0, cursor);
}
@ -98,8 +91,10 @@ public class SqlCompleterTest {
logger.info(explain);
Assert.assertEquals("Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] "
+ explain, expected, newHashSet(candidates));
Assert.assertEquals(
"Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] " + explain,
expected,
newHashSet(candidates));
}
private String explain(String buffer, int cursor, List<InterpreterCompletion> candidates) {
@ -199,8 +194,8 @@ public class SqlCompleterTest {
@Test
public void testFindAliasesInSQL_Simple() {
String sql = "select * from prod_emart.financial_account a";
Map<String, String> res = sqlCompleter.findAliasesInSQL(
delimiter.delimit(sql, 0).getArguments());
Map<String, String> res =
sqlCompleter.findAliasesInSQL(delimiter.delimit(sql, 0).getArguments());
assertEquals(1, res.size());
assertTrue(res.get("a").equals("prod_emart.financial_account"));
}
@ -208,7 +203,8 @@ public class SqlCompleterTest {
@Test
public void testFindAliasesInSQL_Two() {
String sql = "select * from prod_dds.financial_account a, prod_dds.customer b";
Map<String, String> res = sqlCompleter.findAliasesInSQL(
Map<String, String> res =
sqlCompleter.findAliasesInSQL(
sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments());
assertEquals(2, res.size());
assertTrue(res.get("a").equals("prod_dds.financial_account"));
@ -218,7 +214,8 @@ public class SqlCompleterTest {
@Test
public void testFindAliasesInSQL_WrongTables() {
String sql = "select * from prod_ddsxx.financial_account a, prod_dds.customerxx b";
Map<String, String> res = sqlCompleter.findAliasesInSQL(
Map<String, String> res =
sqlCompleter.findAliasesInSQL(
sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments());
assertEquals(0, res.size());
}
@ -231,24 +228,34 @@ public class SqlCompleterTest {
Map<String, String> aliases = new HashMap<>();
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(9, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds",
CompletionType.schema.name())));
assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart",
CompletionType.schema.name())));
assertTrue(candidates.contains(new InterpreterCompletion("SUM", "SUM",
CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("SUBSTRING", "SUBSTRING",
CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN",
CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("SELECT", "SELECT",
CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("ORDER", "ORDER",
CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("LIMIT", "LIMIT",
CompletionType.keyword.name())));
assertTrue(candidates.contains(new InterpreterCompletion("FROM", "FROM",
CompletionType.keyword.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion(
"SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())));
}
@Test
@ -259,10 +266,12 @@ public class SqlCompleterTest {
Map<String, String> aliases = new HashMap<>();
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(2, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds",
CompletionType.schema.name())));
assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart",
CompletionType.schema.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
}
@Test
@ -273,9 +282,10 @@ public class SqlCompleterTest {
Map<String, String> aliases = new HashMap<>();
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(1, candidates.size());
assertTrue(candidates.contains(
new InterpreterCompletion("financial_account", "financial_account",
CompletionType.table.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion(
"financial_account", "financial_account", CompletionType.table.name())));
}
@Test
@ -286,10 +296,12 @@ public class SqlCompleterTest {
Map<String, String> aliases = new HashMap<>();
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(2, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk",
CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id",
CompletionType.column.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
}
@Test
@ -301,10 +313,12 @@ public class SqlCompleterTest {
aliases.put("a", "prod_dds.financial_account");
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(2, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk",
CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id",
CompletionType.column.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
}
@Test
@ -316,71 +330,127 @@ public class SqlCompleterTest {
aliases.put("a", "prod_dds.financial_account");
sqlCompleter.completeName(buffer, cursor, candidates, aliases);
assertEquals(2, candidates.size());
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk",
CompletionType.column.name())));
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id",
CompletionType.column.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
assertTrue(
candidates.contains(
new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
}
@Test
public void testSchemaAndTable() {
String buffer = "select * from prod_emart.fi";
tester.buffer(buffer).from(20).to(23).expect(newHashSet(
new InterpreterCompletion("prod_emart", "prod_emart",
CompletionType.schema.name()))).test();
tester.buffer(buffer).from(25).to(27).expect(newHashSet(
new InterpreterCompletion("financial_account", "financial_account",
CompletionType.table.name()))).test();
tester
.buffer(buffer)
.from(20)
.to(23)
.expect(
newHashSet(
new InterpreterCompletion(
"prod_emart", "prod_emart", CompletionType.schema.name())))
.test();
tester
.buffer(buffer)
.from(25)
.to(27)
.expect(
newHashSet(
new InterpreterCompletion(
"financial_account", "financial_account", CompletionType.table.name())))
.test();
}
@Test
public void testEdges() {
String buffer = " ORDER ";
tester.buffer(buffer).from(3).to(7).expect(newHashSet(
new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(0).to(1).expect(newHashSet(
new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()),
new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN",
CompletionType.keyword.name()),
new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()),
new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()),
new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()),
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()),
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()),
new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())
)).test();
tester
.buffer(buffer)
.from(3)
.to(7)
.expect(
newHashSet(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name())))
.test();
tester
.buffer(buffer)
.from(0)
.to(1)
.expect(
newHashSet(
new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()),
new InterpreterCompletion(
"SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()),
new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()),
new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()),
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()),
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()),
new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())))
.test();
}
@Test
public void testMultipleWords() {
String buffer = "SELE FRO LIM";
tester.buffer(buffer).from(2).to(4).expect(newHashSet(
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(6).to(8).expect(newHashSet(
new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(10).to(12).expect(newHashSet(
new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()))).test();
tester
.buffer(buffer)
.from(2)
.to(4)
.expect(
newHashSet(
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name())))
.test();
tester
.buffer(buffer)
.from(6)
.to(8)
.expect(
newHashSet(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())))
.test();
tester
.buffer(buffer)
.from(10)
.to(12)
.expect(
newHashSet(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name())))
.test();
}
@Test
public void testMultiLineBuffer() {
String buffer = " \n SELE\nFRO";
tester.buffer(buffer).from(5).to(7).expect(newHashSet(
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test();
tester.buffer(buffer).from(9).to(11).expect(newHashSet(
new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test();
tester
.buffer(buffer)
.from(5)
.to(7)
.expect(
newHashSet(
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name())))
.test();
tester
.buffer(buffer)
.from(9)
.to(11)
.expect(
newHashSet(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())))
.test();
}
@Test
public void testMultipleCompletionSuggestions() {
String buffer = "SU";
tester.buffer(buffer).from(2).to(2).expect(newHashSet(
new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN",
CompletionType.keyword.name()),
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()))
).test();
tester
.buffer(buffer)
.from(2)
.to(2)
.expect(
newHashSet(
new InterpreterCompletion(
"SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name())))
.test();
}
@Test

7
kylin/pom.xml
View file

@ -73,13 +73,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

8
kylin/src/main/java/org/apache/zeppelin/kylin/KylinErrorResponse.java
View file

@ -21,9 +21,7 @@ import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import org.apache.zeppelin.common.JsonSerializable;
/**
* class for Kylin Error Response.
*/
/** class for Kylin Error Response. */
class KylinErrorResponse implements JsonSerializable {
private static final Gson gson = new Gson();
@ -34,8 +32,8 @@ class KylinErrorResponse implements JsonSerializable {
private Object data;
private String msg;
KylinErrorResponse(String stacktrace, String exception, String url,
String code, Object data, String msg) {
KylinErrorResponse(
String stacktrace, String exception, String url, String code, Object data, String msg) {
this.stacktrace = stacktrace;
this.exception = exception;
this.url = url;

92
kylin/src/main/java/org/apache/zeppelin/kylin/KylinInterpreter.java
View file

@ -17,6 +17,11 @@
package org.apache.zeppelin.kylin;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
@ -24,23 +29,14 @@ import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.HttpClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Kylin interpreter for Zeppelin. (http://kylin.apache.org)
*/
/** Kylin interpreter for Zeppelin. (http://kylin.apache.org) */
public class KylinInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(KylinInterpreter.class);
@ -53,21 +49,17 @@ public class KylinInterpreter extends Interpreter {
static final String KYLIN_QUERY_ACCEPT_PARTIAL = "kylin.query.ispartial";
static final Pattern KYLIN_TABLE_FORMAT_REGEX_LABEL = Pattern.compile("\"label\":\"(.*?)\"");
static final Pattern KYLIN_TABLE_FORMAT_REGEX_RESULTS =
Pattern.compile("\"results\":\\[\\[(.*?)]]");
Pattern.compile("\"results\":\\[\\[(.*?)]]");
public KylinInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
public void open() {}
@Override
public void close() {
}
public void close() {}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
@ -80,9 +72,7 @@ public class KylinInterpreter extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {
}
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
@ -95,8 +85,8 @@ public class KylinInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return null;
}
@ -109,14 +99,38 @@ public class KylinInterpreter extends Interpreter {
logger.info("acceptPartial:" + getProperty(KYLIN_QUERY_ACCEPT_PARTIAL));
logger.info("limit:" + getProperty(KYLIN_QUERY_LIMIT));
logger.info("offset:" + getProperty(KYLIN_QUERY_OFFSET));
byte[] encodeBytes = Base64.encodeBase64(new String(getProperty(KYLIN_USERNAME)
+ ":" + getProperty(KYLIN_PASSWORD)).getBytes("UTF-8"));
byte[] encodeBytes =
Base64.encodeBase64(
new String(getProperty(KYLIN_USERNAME) + ":" + getProperty(KYLIN_PASSWORD))
.getBytes("UTF-8"));
String postContent = new String("{\"project\":" + "\"" + kylinProject + "\""
+ "," + "\"sql\":" + "\"" + kylinSql + "\""
+ "," + "\"acceptPartial\":" + "\"" + getProperty(KYLIN_QUERY_ACCEPT_PARTIAL) + "\""
+ "," + "\"offset\":" + "\"" + getProperty(KYLIN_QUERY_OFFSET) + "\""
+ "," + "\"limit\":" + "\"" + getProperty(KYLIN_QUERY_LIMIT) + "\"" + "}");
String postContent =
new String(
"{\"project\":"
+ "\""
+ kylinProject
+ "\""
+ ","
+ "\"sql\":"
+ "\""
+ kylinSql
+ "\""
+ ","
+ "\"acceptPartial\":"
+ "\""
+ getProperty(KYLIN_QUERY_ACCEPT_PARTIAL)
+ "\""
+ ","
+ "\"offset\":"
+ "\""
+ getProperty(KYLIN_QUERY_OFFSET)
+ "\""
+ ","
+ "\"limit\":"
+ "\""
+ getProperty(KYLIN_QUERY_LIMIT)
+ "\""
+ "}");
logger.info("post:" + postContent);
postContent = postContent.replaceAll("[\u0000-\u001f]", " ");
StringEntity entity = new StringEntity(postContent, "UTF-8");
@ -182,9 +196,10 @@ public class KylinInterpreter extends Interpreter {
logger.error("Cannot get json from string: " + result);
// when code is 401, the response is html, not json
if (code == 401) {
errorMessage.append(" Error message: Unauthorized. This request requires "
+ "HTTP authentication. Please make sure your have set your credentials "
+ "correctly.");
errorMessage.append(
" Error message: Unauthorized. This request requires "
+ "HTTP authentication. Please make sure your have set your credentials "
+ "correctly.");
} else {
errorMessage.append(" Error message: " + result + " .");
}
@ -200,19 +215,18 @@ public class KylinInterpreter extends Interpreter {
throw new IOException(e);
}
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
formatResult(result));
return new InterpreterResult(InterpreterResult.Code.SUCCESS, formatResult(result));
}
String formatResult(String msg) {
StringBuilder res = new StringBuilder("%table ");
Matcher ml = KYLIN_TABLE_FORMAT_REGEX_LABEL.matcher(msg);
while (!ml.hitEnd() && ml.find()) {
res.append(ml.group(1) + " \t");
}
}
res.append(" \n");
Matcher mr = KYLIN_TABLE_FORMAT_REGEX_RESULTS.matcher(msg);
String table = null;
while (!mr.hitEnd() && mr.find()) {

136
kylin/src/test/java/org/apache/zeppelin/kylin/KylinInterpreterTest.java
View file

@ -19,6 +19,12 @@ package org.apache.zeppelin.kylin;
import static org.junit.Assert.assertEquals;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Locale;
import java.util.Properties;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@ -26,19 +32,11 @@ import org.apache.http.ProtocolVersion;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.AbstractHttpMessage;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Locale;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
public class KylinInterpreterTest {
static final Properties KYLIN_PROPERTIES = new Properties();
@ -56,38 +54,54 @@ public class KylinInterpreterTest {
@Test
public void testWithDefault() {
KylinInterpreter t = new MockKylinInterpreter(getDefaultProperties());
InterpreterResult result = t.interpret(
"select a.date,sum(b.measure) as measure from kylin_fact_table a " +
"inner join kylin_lookup_table b on a.date=b.date group by a.date", null);
assertEquals("default", t.getProject("select a.date,sum(b.measure) as measure "
+ "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+ "group by a.date"));
InterpreterResult result =
t.interpret(
"select a.date,sum(b.measure) as measure from kylin_fact_table a "
+ "inner join kylin_lookup_table b on a.date=b.date group by a.date",
null);
assertEquals(
"default",
t.getProject(
"select a.date,sum(b.measure) as measure "
+ "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+ "group by a.date"));
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
}
@Test
public void testWithProject() {
KylinInterpreter t = new MockKylinInterpreter(getDefaultProperties());
assertEquals("project2", t.getProject("(project2)\n select a.date,sum(b.measure) "
+ "as measure from kylin_fact_table a inner join kylin_lookup_table b on "
+ "a.date=b.date group by a.date"));
assertEquals("", t.getProject("()\n select a.date,sum(b.measure) as measure "
+ "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+ "group by a.date"));
assertEquals("\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
assertEquals(
"project2",
t.getProject(
"(project2)\n select a.date,sum(b.measure) "
+ "as measure from kylin_fact_table a inner join kylin_lookup_table b on "
+ "a.date=b.date group by a.date"));
assertEquals(
"",
t.getProject(
"()\n select a.date,sum(b.measure) as measure "
+ "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+ "group by a.date"));
assertEquals(
"\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
+ "inner join kylin_lookup_table b on a.date=b.date group by a.date",
t.getSQL("(project2)\n select a.date,sum(b.measure) as measure "
+ "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+ "group by a.date"));
assertEquals("\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
t.getSQL(
"(project2)\n select a.date,sum(b.measure) as measure "
+ "from kylin_fact_table a inner join kylin_lookup_table b on a.date=b.date "
+ "group by a.date"));
assertEquals(
"\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
+ "inner join kylin_lookup_table b on a.date=b.date group by a.date",
t.getSQL("()\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
+ "inner join kylin_lookup_table b on a.date=b.date group by a.date"));
t.getSQL(
"()\n select a.date,sum(b.measure) as measure from kylin_fact_table a "
+ "inner join kylin_lookup_table b on a.date=b.date group by a.date"));
}
@Test
public void testParseResult() {
String msg = "{\"columnMetas\":[{\"isNullable\":1,\"displaySize\":256,\"label\":\"COUNTRY\","
String msg =
"{\"columnMetas\":[{\"isNullable\":1,\"displaySize\":256,\"label\":\"COUNTRY\","
+ "\"name\":\"COUNTRY\",\"schemaName\":\"DEFAULT\",\"catelogName\":null,"
+ "\"tableName\":\"SALES_TABLE\",\"precision\":256,\"scale\":0,\"columnType\":12,"
+ "\"columnTypeName\":\"VARCHAR\",\"writable\":false,\"readOnly\":true,"
@ -109,11 +123,12 @@ public class KylinInterpreterTest {
+ "\"isException\":false,\"exceptionMessage\":null,\"duration\":134,"
+ "\"totalScanCount\":1,\"hitExceptionCache\":false,\"storageCacheUsed\":false,"
+ "\"partial\":false}";
String expected = "%table COUNTRY \tCURRENCY \tCOUNT__ \t \n" +
"AMERICA \tUSD \tnull \t \n" +
"null \tRMB \t0 \t \n" +
"KOR \tnull \t100 \t \n" +
"\\\"abc\\\" \ta,b,c \t-1 \t \n";
String expected =
"%table COUNTRY \tCURRENCY \tCOUNT__ \t \n"
+ "AMERICA \tUSD \tnull \t \n"
+ "null \tRMB \t0 \t \n"
+ "KOR \tnull \t100 \t \n"
+ "\\\"abc\\\" \ta,b,c \t-1 \t \n";
KylinInterpreter t = new MockKylinInterpreter(getDefaultProperties());
String actual = t.formatResult(msg);
Assert.assertEquals(expected, actual);
@ -121,7 +136,8 @@ public class KylinInterpreterTest {
@Test
public void testParseEmptyResult() {
String msg = "{\"columnMetas\":[{\"isNullable\":1,\"displaySize\":256,\"label\":\"COUNTRY\","
String msg =
"{\"columnMetas\":[{\"isNullable\":1,\"displaySize\":256,\"label\":\"COUNTRY\","
+ "\"name\":\"COUNTRY\",\"schemaName\":\"DEFAULT\",\"catelogName\":null,"
+ "\"tableName\":\"SALES_TABLE\",\"precision\":256,\"scale\":0,\"columnType\":12,"
+ "\"columnTypeName\":\"VARCHAR\",\"writable\":false,\"readOnly\":true,"
@ -138,7 +154,8 @@ public class KylinInterpreterTest {
+ "\"columnType\":-5,\"columnTypeName\":\"BIGINT\",\"writable\":false,"
+ "\"readOnly\":true,\"definitelyWritable\":false,\"autoIncrement\":false,"
+ "\"caseSensitive\":true,\"searchable\":false,\"currency\":false,\"signed\":true}],"
+ "\"results\":[]," + "\"cube\":\"Sample_Cube\",\"affectedRowCount\":0,"
+ "\"results\":[],"
+ "\"cube\":\"Sample_Cube\",\"affectedRowCount\":0,"
+ "\"isException\":false,\"exceptionMessage\":null,\"duration\":134,"
+ "\"totalScanCount\":1,\"hitExceptionCache\":false,\"storageCacheUsed\":false,"
+ "\"partial\":false}";
@ -173,37 +190,32 @@ class MockKylinInterpreter extends KylinInterpreter {
}
}
class MockHttpClient{
public MockHttpResponse execute(HttpPost post){
class MockHttpClient {
public MockHttpResponse execute(HttpPost post) {
return new MockHttpResponse();
}
}
class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
class MockHttpResponse extends AbstractHttpMessage implements HttpResponse {
@Override
public StatusLine getStatusLine() {
return new MockStatusLine();
}
@Override
public void setStatusLine(StatusLine statusLine) {
}
public void setStatusLine(StatusLine statusLine) {}
@Override
public void setStatusLine(ProtocolVersion protocolVersion, int i) {
}
public void setStatusLine(ProtocolVersion protocolVersion, int i) {}
@Override
public void setStatusLine(ProtocolVersion protocolVersion, int i, String s) {
}
public void setStatusLine(ProtocolVersion protocolVersion, int i, String s) {}
@Override
public void setStatusCode(int i) throws IllegalStateException {
}
public void setStatusCode(int i) throws IllegalStateException {}
@Override
public void setReasonPhrase(String s) throws IllegalStateException {
}
public void setReasonPhrase(String s) throws IllegalStateException {}
@Override
public HttpEntity getEntity() {
@ -211,8 +223,7 @@ class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
}
@Override
public void setEntity(HttpEntity httpEntity) {
}
public void setEntity(HttpEntity httpEntity) {}
@Override
public Locale getLocale() {
@ -220,8 +231,7 @@ class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
}
@Override
public void setLocale(Locale locale) {
}
public void setLocale(Locale locale) {}
@Override
public ProtocolVersion getProtocolVersion() {
@ -229,7 +239,7 @@ class MockHttpResponse extends AbstractHttpMessage implements HttpResponse{
}
}
class MockStatusLine implements StatusLine{
class MockStatusLine implements StatusLine {
@Override
public ProtocolVersion getProtocolVersion() {
return null;
@ -246,7 +256,7 @@ class MockStatusLine implements StatusLine{
}
}
class MockEntity implements HttpEntity{
class MockEntity implements HttpEntity {
@Override
public boolean isRepeatable() {
return false;
@ -274,15 +284,16 @@ class MockEntity implements HttpEntity{
@Override
public InputStream getContent() throws IOException, IllegalStateException {
return new ByteArrayInputStream(("{\"columnMetas\":" +
"[{\"label\":\"PART_DT\"},{\"label\":\"measure\"}]," +
"\"results\":[[\"2012-01-03\",\"917.4138\"]," +
"[\"2012-05-06\",\"592.4823\"]]}").getBytes());
return new ByteArrayInputStream(
("{\"columnMetas\":"
+ "[{\"label\":\"PART_DT\"},{\"label\":\"measure\"}],"
+ "\"results\":[[\"2012-01-03\",\"917.4138\"],"
+ "[\"2012-05-06\",\"592.4823\"]]}")
.getBytes());
}
@Override
public void writeTo(OutputStream outputStream) throws IOException {
}
public void writeTo(OutputStream outputStream) throws IOException {}
@Override
public boolean isStreaming() {
@ -290,6 +301,5 @@ class MockEntity implements HttpEntity{
}
@Override
public void consumeContent() throws IOException {
}
public void consumeContent() throws IOException {}
}

7
lens/pom.xml
View file

@ -171,13 +171,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

25
lens/src/main/java/org/apache/zeppelin/lens/ExecutionDetail.java
View file

@ -1,18 +1,15 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
@ -20,9 +17,7 @@ package org.apache.zeppelin.lens;
import org.apache.lens.client.LensClient;
import org.springframework.shell.core.JLineShell;
/**
* Pojo tracking query execution details. Used to cancel the query.
*/
/** Pojo tracking query execution details. Used to cancel the query. */
public class ExecutionDetail {
private String queryHandle;
private LensClient lensClient;

13
lens/src/main/java/org/apache/zeppelin/lens/LensBootstrap.java
View file

@ -1,12 +1,12 @@
/*
* Copyright 2011-2012 the original author or authors.
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -19,14 +19,13 @@ package org.apache.zeppelin.lens;
import org.springframework.beans.factory.support.DefaultListableBeanFactory;
import org.springframework.beans.factory.support.RootBeanDefinition;
import org.springframework.context.support.GenericApplicationContext;
/**
* workaround for https://github.com/spring-projects/spring-shell/issues/73.
*/
/** workaround for https://github.com/spring-projects/spring-shell/issues/73. */
public class LensBootstrap extends org.springframework.shell.Bootstrap {
public LensBootstrap() {
super();
}
public LensJLineShellComponent getLensJLineShellComponent() {
GenericApplicationContext ctx = (GenericApplicationContext) getApplicationContext();
RootBeanDefinition rbd = new RootBeanDefinition();

174
lens/src/main/java/org/apache/zeppelin/lens/LensInterpreter.java
View file

@ -1,36 +1,19 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
import org.apache.lens.cli.commands.BaseLensCommand;
import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientConfig;
import org.apache.lens.client.LensClientSingletonWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.shell.Bootstrap;
import org.springframework.shell.core.CommandResult;
import org.springframework.shell.core.JLineShell;
import org.springframework.shell.core.JLineShellComponent;
import org.springframework.shell.support.logging.HandlerUtils;
import java.io.ByteArrayOutputStream;
import java.util.LinkedHashMap;
import java.util.List;
@ -39,7 +22,10 @@ import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lens.cli.commands.BaseLensCommand;
import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientConfig;
import org.apache.lens.client.LensClientSingletonWrapper;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -47,10 +33,15 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.shell.Bootstrap;
import org.springframework.shell.core.CommandResult;
import org.springframework.shell.core.JLineShell;
import org.springframework.shell.core.JLineShellComponent;
import org.springframework.shell.support.logging.HandlerUtils;
/**
* Lens interpreter for Zeppelin.
*/
/** Lens interpreter for Zeppelin. */
public class LensInterpreter extends Interpreter {
static final Logger LOGGER = LoggerFactory.getLogger(LensInterpreter.class);
static final String LENS_CLIENT_DBNAME = "lens.client.dbname";
@ -60,22 +51,23 @@ public class LensInterpreter extends Interpreter {
static final String ZEPPELIN_LENS_RUN_CONCURRENT_SESSION = "zeppelin.lens.run.concurrent";
static final String ZEPPELIN_LENS_CONCURRENT_SESSIONS = "zeppelin.lens.maxThreads";
static final String ZEPPELIN_MAX_ROWS = "zeppelin.lens.maxResults";
static final Map<String, Pattern> LENS_TABLE_FORMAT_REGEX = new LinkedHashMap<String, Pattern>() {
{
put("cubes", Pattern.compile(".*show\\s+cube.*"));
put("nativetables", Pattern.compile(".*show\\s+nativetable.*"));
put("storages", Pattern.compile(".*show\\s+storage.*"));
put("facts", Pattern.compile(".*show\\s+fact.*"));
put("dimensions", Pattern.compile(".*show\\s+dimension.*"));
put("params", Pattern.compile(".*show\\s+param.*"));
put("databases", Pattern.compile(".*show\\s+database.*"));
put("query results", Pattern.compile(".*query\\s+results.*"));
}
};
static final Map<String, Pattern> LENS_TABLE_FORMAT_REGEX =
new LinkedHashMap<String, Pattern>() {
{
put("cubes", Pattern.compile(".*show\\s+cube.*"));
put("nativetables", Pattern.compile(".*show\\s+nativetable.*"));
put("storages", Pattern.compile(".*show\\s+storage.*"));
put("facts", Pattern.compile(".*show\\s+fact.*"));
put("dimensions", Pattern.compile(".*show\\s+dimension.*"));
put("params", Pattern.compile(".*show\\s+param.*"));
put("databases", Pattern.compile(".*show\\s+database.*"));
put("query results", Pattern.compile(".*query\\s+results.*"));
}
};
private static Pattern queryExecutePattern = Pattern.compile(".*query\\s+execute\\s+(.*)");
// tracks paragraphId -> Lens QueryHandle
private static Map<String, ExecutionDetail> paraToQH = new ConcurrentHashMap<> ();
private static Map<String, ExecutionDetail> paraToQH = new ConcurrentHashMap<>();
private static Map<LensClient, Boolean> clientMap = new ConcurrentHashMap<>();
private int maxResults;
@ -91,22 +83,26 @@ public class LensInterpreter extends Interpreter {
lensClientConfig = new LensClientConfig();
lensClientConfig.set(LENS_SERVER_URL, property.get(LENS_SERVER_URL).toString());
lensClientConfig.set(LENS_CLIENT_DBNAME, property.get(LENS_CLIENT_DBNAME).toString());
lensClientConfig.set(LENS_SESSION_CLUSTER_USER, property.get(LENS_SESSION_CLUSTER_USER)
.toString());
lensClientConfig.set(
LENS_SESSION_CLUSTER_USER, property.get(LENS_SESSION_CLUSTER_USER).toString());
lensClientConfig.set(LENS_PERSIST_RESULTSET, property.get(LENS_PERSIST_RESULTSET).toString());
try {
maxResults = Integer.parseInt(property.get(ZEPPELIN_MAX_ROWS).toString());
} catch (NumberFormatException | NullPointerException e) {
maxResults = 1000;
LOGGER.error("unable to parse " + ZEPPELIN_MAX_ROWS + " :"
+ property.get(ZEPPELIN_MAX_ROWS), e);
LOGGER.error(
"unable to parse " + ZEPPELIN_MAX_ROWS + " :" + property.get(ZEPPELIN_MAX_ROWS), e);
}
try {
maxThreads = Integer.parseInt(property.get(ZEPPELIN_LENS_CONCURRENT_SESSIONS).toString());
} catch (NumberFormatException | NullPointerException e) {
maxThreads = 10;
LOGGER.error("unable to parse " + ZEPPELIN_LENS_CONCURRENT_SESSIONS + " :"
+ property.get(ZEPPELIN_LENS_CONCURRENT_SESSIONS), e);
LOGGER.error(
"unable to parse "
+ ZEPPELIN_LENS_CONCURRENT_SESSIONS
+ " :"
+ property.get(ZEPPELIN_LENS_CONCURRENT_SESSIONS),
e);
}
LOGGER.info("LensInterpreter created");
} catch (Exception e) {
@ -144,7 +140,7 @@ public class LensInterpreter extends Interpreter {
init();
LOGGER.info("LensInterpreter opened");
}
@Override
public void close() {
closeConnections();
@ -171,11 +167,10 @@ public class LensInterpreter extends Interpreter {
LensClient lensClient;
try {
lensClient = new LensClient(lensClientConfig);
for (String beanName : bs.getApplicationContext().getBeanDefinitionNames()) {
if (bs.getApplicationContext().getBean(beanName) instanceof BaseLensCommand) {
((BaseLensCommand) bs.getApplicationContext().getBean(beanName))
.setClient(lensClient);
((BaseLensCommand) bs.getApplicationContext().getBean(beanName)).setClient(lensClient);
}
}
} catch (Exception e) {
@ -188,11 +183,12 @@ public class LensInterpreter extends Interpreter {
private InterpreterResult handleHelp(JLineShell shell, String st) {
java.util.logging.StreamHandler sh = null;
java.util.logging.Logger springLogger = null;
java.util.logging.Formatter formatter = new java.util.logging.Formatter() {
public String format(java.util.logging.LogRecord record) {
return record.getMessage();
}
};
java.util.logging.Formatter formatter =
new java.util.logging.Formatter() {
public String format(java.util.logging.LogRecord record) {
return record.getMessage();
}
};
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
sh = new java.util.logging.StreamHandler(baos, formatter);
@ -209,7 +205,7 @@ public class LensInterpreter extends Interpreter {
}
return new InterpreterResult(Code.SUCCESS, baos.toString());
}
private String modifyQueryStatement(String st) {
Matcher matcher = queryExecutePattern.matcher(st.toLowerCase());
if (!matcher.find()) {
@ -234,36 +230,40 @@ public class LensInterpreter extends Interpreter {
}
String st = input.replaceAll("\\n", " ");
LOGGER.info("LensInterpreter command: " + st);
Bootstrap bs = createBootstrap();
JLineShell shell = getJLineShell(bs);
JLineShell shell = getJLineShell(bs);
CommandResult res;
LensClient lensClient = null;
String qh = null;
if (st.trim().startsWith("help")) {
return handleHelp(shell, st);
}
try {
lensClient = createAndSetLensClient(bs);
clientMap.put(lensClient, true);
String lensCommand = modifyQueryStatement(st);
LOGGER.info("executing command : " + lensCommand);
res = shell.executeCommand(lensCommand);
if (!lensCommand.equals(st) && res != null
&& res.getResult() != null
if (!lensCommand.equals(st)
&& res != null
&& res.getResult() != null
&& res.getResult().toString().trim().matches("[a-z0-9-]+")) {
// setup query progress tracking
qh = res.getResult().toString();
paraToQH.put(context.getParagraphId(), new ExecutionDetail(qh, lensClient, shell));
String getResultsCmd = "query results --async false " + qh;
LOGGER.info("executing query results command : " + context.getParagraphId() + " : "
LOGGER.info(
"executing query results command : "
+ context.getParagraphId()
+ " : "
+ getResultsCmd);
res = shell.executeCommand(getResultsCmd);
res = shell.executeCommand(getResultsCmd);
paraToQH.remove(context.getParagraphId());
}
} catch (Exception ex) {
@ -283,7 +283,7 @@ public class LensInterpreter extends Interpreter {
}
return new InterpreterResult(Code.SUCCESS, formatResult(st, res));
}
private void closeShell(JLineShell shell) {
if (shell instanceof LensJLineShellComponent) {
((LensJLineShellComponent) shell).stop();
@ -291,7 +291,7 @@ public class LensInterpreter extends Interpreter {
((JLineShellComponent) shell).stop();
}
}
private String formatResult(String st, CommandResult result) {
if (result == null) {
return "error in interpret, no result object returned";
@ -299,8 +299,8 @@ public class LensInterpreter extends Interpreter {
if (!result.isSuccess() || result.getResult() == null) {
if (result.getException() != null) {
return result.getException().getMessage();
//try describe cube (without cube name)- error is written as a warning,
//but not returned to result object
// try describe cube (without cube name)- error is written as a warning,
// but not returned to result object
} else {
return "error in interpret, unable to execute command";
}
@ -312,15 +312,15 @@ public class LensInterpreter extends Interpreter {
break;
}
}
if (queryExecutePattern.matcher(st.toLowerCase()).find() &&
result.getResult().toString().contains(" rows process in (")) {
if (queryExecutePattern.matcher(st.toLowerCase()).find()
&& result.getResult().toString().contains(" rows process in (")) {
sb.append("%table ");
}
if (sb.length() > 0) {
return sb.append(result.getResult().toString()).toString();
}
return result.getResult().toString();
//Lens sends error messages without setting result.isSuccess() = false.
// Lens sends error messages without setting result.isSuccess() = false.
}
@Override
@ -339,7 +339,12 @@ public class LensInterpreter extends Interpreter {
clientMap.put(lensClient, true);
LOGGER.info("invoke query kill (" + context.getParagraphId() + ") " + qh);
CommandResult res = shell.executeCommand("query kill " + qh);
LOGGER.info("query kill returned (" + context.getParagraphId() + ") " + qh + " with: "
LOGGER.info(
"query kill returned ("
+ context.getParagraphId()
+ ") "
+ qh
+ " with: "
+ res.getResult());
} catch (Exception e) {
LOGGER.error("unable to kill query (" + context.getParagraphId() + ") " + qh, e);
@ -380,7 +385,7 @@ public class LensInterpreter extends Interpreter {
clientMap.put(lensClient, true);
CommandResult res = shell.executeCommand("query status " + qh);
LOGGER.info(context.getParagraphId() + " --> " + res.getResult().toString());
//change to debug
// change to debug
Pattern pattern = Pattern.compile(".*(Progress : (\\d\\.\\d)).*");
Matcher matcher = pattern.matcher(res.getResult().toString().replaceAll("\\n", " "));
if (matcher.find(2)) {
@ -410,11 +415,11 @@ public class LensInterpreter extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return null;
}
public boolean concurrentRequests() {
return Boolean.parseBoolean(getProperty(ZEPPELIN_LENS_RUN_CONCURRENT_SESSION));
}
@ -422,8 +427,9 @@ public class LensInterpreter extends Interpreter {
@Override
public Scheduler getScheduler() {
if (concurrentRequests()) {
return SchedulerFactory.singleton().createOrGetParallelScheduler(
LensInterpreter.class.getName() + this.hashCode(), maxThreads);
return SchedulerFactory.singleton()
.createOrGetParallelScheduler(
LensInterpreter.class.getName() + this.hashCode(), maxThreads);
} else {
return super.getScheduler();
}

90
lens/src/main/java/org/apache/zeppelin/lens/LensJLineShellComponent.java
View file

@ -1,12 +1,12 @@
/*
* Copyright 2011-2012 the original author or authors.
*
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -15,9 +15,9 @@
*/
package org.apache.zeppelin.lens;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactoryUtils;
import org.springframework.beans.factory.InitializingBean;
@ -37,16 +37,11 @@ import org.springframework.shell.plugin.HistoryFileNameProvider;
import org.springframework.shell.plugin.PluginUtils;
import org.springframework.shell.plugin.PromptProvider;
import java.util.Map;
/** workaround for https://github.com/spring-projects/spring-shell/issues/73. */
public class LensJLineShellComponent extends JLineShell
implements SmartLifecycle, ApplicationContextAware, InitializingBean {
@Autowired private CommandLine commandLine;
/**
* workaround for https://github.com/spring-projects/spring-shell/issues/73.
*/
public class LensJLineShellComponent extends JLineShell implements SmartLifecycle,
ApplicationContextAware, InitializingBean {
@Autowired
private CommandLine commandLine;
private volatile boolean running = false;
private Thread shellThread;
@ -72,18 +67,18 @@ public class LensJLineShellComponent extends JLineShell implements SmartLifecycl
public boolean isAutoStartup() {
return false;
}
public void stop(Runnable callback) {
stop();
callback.run();
}
public int getPhase() {
return 1;
}
public void start() {
//customizePlug must run before start thread to take plugin's configuration into effect
// customizePlug must run before start thread to take plugin's configuration into effect
customizePlugin();
shellThread = new Thread(this, "Spring Shell");
shellThread.start();
@ -100,31 +95,29 @@ public class LensJLineShellComponent extends JLineShell implements SmartLifecycl
public boolean isRunning() {
return running;
}
@SuppressWarnings("rawtypes")
public void afterPropertiesSet() {
Map<String, CommandMarker> commands = BeanFactoryUtils.beansOfTypeIncludingAncestors(
applicationContext, CommandMarker.class);
Map<String, CommandMarker> commands =
BeanFactoryUtils.beansOfTypeIncludingAncestors(applicationContext, CommandMarker.class);
for (CommandMarker command : commands.values()) {
getSimpleParser().add(command);
}
Map<String, Converter> converters = BeanFactoryUtils.beansOfTypeIncludingAncestors(
applicationContext, Converter.class);
Map<String, Converter> converters =
BeanFactoryUtils.beansOfTypeIncludingAncestors(applicationContext, Converter.class);
for (Converter<?> converter : converters.values()) {
getSimpleParser().add(converter);
}
setHistorySize(commandLine.getHistorySize());
if (commandLine.getShellCommandsToExecute() != null) {
setPrintBanner(false);
}
}
/**
* Wait the shell command to complete by typing "quit" or "exit".
*/
/** Wait the shell command to complete by typing "quit" or "exit". */
public void waitForComplete() {
try {
shellThread.join();
@ -163,13 +156,14 @@ public class LensJLineShellComponent extends JLineShell implements SmartLifecycl
}
/**
* get history file name from provider. The provider has highest order
* {@link org.springframework.core.Ordered#getOrder()} will win.
*
* @return history file name
* get history file name from provider. The provider has highest order {@link
* org.springframework.core.Ordered#getOrder()} will win.
*
* @return history file name
*/
protected String getHistoryFileName() {
HistoryFileNameProvider historyFileNameProvider = PluginUtils.getHighestPriorityProvider(
HistoryFileNameProvider historyFileNameProvider =
PluginUtils.getHighestPriorityProvider(
this.applicationContext, HistoryFileNameProvider.class);
String providerHistoryFileName = historyFileNameProvider.getHistoryFileName();
if (providerHistoryFileName != null) {
@ -180,14 +174,14 @@ public class LensJLineShellComponent extends JLineShell implements SmartLifecycl
}
/**
* get prompt text from provider. The provider has highest order
* {@link org.springframework.core.Ordered#getOrder()} will win.
*
* get prompt text from provider. The provider has highest order {@link
* org.springframework.core.Ordered#getOrder()} will win.
*
* @return prompt text
*/
protected String getPromptText() {
PromptProvider promptProvider = PluginUtils.getHighestPriorityProvider(this.applicationContext,
PromptProvider.class);
PromptProvider promptProvider =
PluginUtils.getHighestPriorityProvider(this.applicationContext, PromptProvider.class);
String providerPromptText = promptProvider.getPrompt();
if (providerPromptText != null) {
return providerPromptText;
@ -197,18 +191,16 @@ public class LensJLineShellComponent extends JLineShell implements SmartLifecycl
}
/**
* Get Banner and Welcome Message from provider. The provider has highest order
* {@link org.springframework.core.Ordered#getOrder()} will win.
* Get Banner and Welcome Message from provider. The provider has highest order {@link
* org.springframework.core.Ordered#getOrder()} will win.
*
* @return BannerText[0]: Banner
* BannerText[1]: Welcome Message
* BannerText[2]: Version
* BannerText[3]: Product Name
* @return BannerText[0]: Banner BannerText[1]: Welcome Message BannerText[2]: Version
* BannerText[3]: Product Name
*/
private String[] getBannerText() {
String[] bannerText = new String[4];
BannerProvider provider = PluginUtils.getHighestPriorityProvider(this.applicationContext,
BannerProvider.class);
BannerProvider provider =
PluginUtils.getHighestPriorityProvider(this.applicationContext, BannerProvider.class);
bannerText[0] = provider.getBanner();
bannerText[1] = provider.getWelcomeMessage();
bannerText[2] = provider.getVersion();
@ -225,24 +217,22 @@ public class LensJLineShellComponent extends JLineShell implements SmartLifecycl
/**
* get the welcome message at start.
*
*
* @return welcome message
*/
public String getWelcomeMessage() {
return this.welcomeMessage;
}
/**
* @param printBanner the printBanner to set
*/
/** @param printBanner the printBanner to set */
public void setPrintBanner(boolean printBanner) {
this.printBanner = printBanner;
}
protected String getProductName() {
return productName;
}
protected String getVersion() {
return version;
}

11
lens/src/main/java/org/apache/zeppelin/lens/LensSimpleExecutionStrategy.java
View file

@ -15,6 +15,7 @@
*/
package org.apache.zeppelin.lens;
import java.util.logging.Logger;
import org.springframework.shell.core.ExecutionProcessor;
import org.springframework.shell.core.ExecutionStrategy;
import org.springframework.shell.event.ParseResult;
@ -22,11 +23,7 @@ import org.springframework.shell.support.logging.HandlerUtils;
import org.springframework.util.Assert;
import org.springframework.util.ReflectionUtils;
import java.util.logging.Logger;
/**
* workaround for https://github.com/spring-projects/spring-shell/issues/73.
*/
/** workaround for https://github.com/spring-projects/spring-shell/issues/73. */
public class LensSimpleExecutionStrategy implements ExecutionStrategy {
private static final Logger logger = HandlerUtils.getLogger(LensSimpleExecutionStrategy.class);
@ -56,8 +53,8 @@ public class LensSimpleExecutionStrategy implements ExecutionStrategy {
private Object invoke(ParseResult parseResult) {
try {
return ReflectionUtils.invokeMethod(parseResult.getMethod(),
parseResult.getInstance(), parseResult.getArguments());
return ReflectionUtils.invokeMethod(
parseResult.getMethod(), parseResult.getInstance(), parseResult.getArguments());
} catch (Throwable th) {
logger.severe("Command failed " + th);
return handleThrowable(th);

48
lens/src/test/java/org/apache/zeppelin/lens/LensInterpreterTest.java
View file

@ -1,24 +1,19 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.lens;
import static org.junit.Assert.assertEquals;
import static org.apache.zeppelin.lens.LensInterpreter.LENS_CLIENT_DBNAME;
import static org.apache.zeppelin.lens.LensInterpreter.LENS_PERSIST_RESULTSET;
import static org.apache.zeppelin.lens.LensInterpreter.LENS_SERVER_URL;
@ -26,26 +21,21 @@ import static org.apache.zeppelin.lens.LensInterpreter.LENS_SESSION_CLUSTER_USER
import static org.apache.zeppelin.lens.LensInterpreter.ZEPPELIN_LENS_CONCURRENT_SESSIONS;
import static org.apache.zeppelin.lens.LensInterpreter.ZEPPELIN_LENS_RUN_CONCURRENT_SESSION;
import static org.apache.zeppelin.lens.LensInterpreter.ZEPPELIN_MAX_ROWS;
import static org.junit.Assert.assertEquals;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.Properties;
import org.apache.zeppelin.interpreter.InterpreterResult;
/**
* Lens interpreter unit tests.
*/
/** Lens interpreter unit tests. */
public class LensInterpreterTest {
@Before
public void setUp() throws Exception {
}
public void setUp() throws Exception {}
@After
public void tearDown() throws Exception {
}
public void tearDown() throws Exception {}
@Test
public void test() {
@ -59,17 +49,17 @@ public class LensInterpreterTest {
prop.setProperty(ZEPPELIN_LENS_CONCURRENT_SESSIONS, "10");
LensInterpreter t = new MockLensInterpreter(prop);
t.open();
//simple help test
// simple help test
InterpreterResult result = t.interpret("help", null);
assertEquals(result.message().get(0).getType(), InterpreterResult.Type.TEXT);
//assertEquals("unable to find 'query execute' in help message",
// assertEquals("unable to find 'query execute' in help message",
// result.message().contains("query execute"), result.message());
t.close();
}
class MockLensInterpreter extends LensInterpreter {
MockLensInterpreter(Properties property) {
super(property);
super(property);
}
@Override

8
livy/pom.xml
View file

@ -407,14 +407,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

11
livy/src/main/java/org/apache/zeppelin/livy/APINotFoundException.java
View file

@ -17,12 +17,9 @@
package org.apache.zeppelin.livy;
/**
* APINotFoundException happens because we may introduce new apis in new livy version.
*/
/** APINotFoundException happens because we may introduce new apis in new livy version. */
public class APINotFoundException extends LivyException {
public APINotFoundException() {
}
public APINotFoundException() {}
public APINotFoundException(String message) {
super(message);
@ -36,8 +33,8 @@ public class APINotFoundException extends LivyException {
super(cause);
}
public APINotFoundException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
public APINotFoundException(
String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

386
livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
View file

@ -20,7 +20,22 @@ package org.apache.zeppelin.livy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.annotations.SerializedName;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.KeyStore;
import java.security.Principal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.net.ssl.SSLContext;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.http.auth.AuthSchemeProvider;
@ -38,6 +53,13 @@ import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
@ -53,35 +75,7 @@ import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.security.KeyStore;
import java.security.Principal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.net.ssl.SSLContext;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
/**
* Base class for livy interpreters.
*/
/** Base class for livy interpreters. */
public abstract class BaseLivyInterpreter extends Interpreter {
protected static final Logger LOGGER = LoggerFactory.getLogger(BaseLivyInterpreter.class);
@ -102,32 +96,34 @@ public abstract class BaseLivyInterpreter extends Interpreter {
// delegate to sharedInterpreter when it is available
protected LivySharedInterpreter sharedInterpreter;
Set<Object> paragraphsToCancel = Collections.newSetFromMap(
new ConcurrentHashMap<Object, Boolean>());
Set<Object> paragraphsToCancel =
Collections.newSetFromMap(new ConcurrentHashMap<Object, Boolean>());
private ConcurrentHashMap<String, Integer> paragraphId2StmtProgressMap =
new ConcurrentHashMap<>();
public BaseLivyInterpreter(Properties property) {
super(property);
this.livyURL = property.getProperty("zeppelin.livy.url");
this.displayAppInfo = Boolean.parseBoolean(
property.getProperty("zeppelin.livy.displayAppInfo", "true"));
this.restartDeadSession = Boolean.parseBoolean(
property.getProperty("zeppelin.livy.restart_dead_session", "false"));
this.sessionCreationTimeout = Integer.parseInt(
property.getProperty("zeppelin.livy.session.create_timeout", 120 + ""));
this.pullStatusInterval = Integer.parseInt(
property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + ""));
this.maxLogLines = Integer.parseInt(property.getProperty("zeppelin.livy.maxLogLines",
"1000"));
this.displayAppInfo =
Boolean.parseBoolean(property.getProperty("zeppelin.livy.displayAppInfo", "true"));
this.restartDeadSession =
Boolean.parseBoolean(property.getProperty("zeppelin.livy.restart_dead_session", "false"));
this.sessionCreationTimeout =
Integer.parseInt(property.getProperty("zeppelin.livy.session.create_timeout", 120 + ""));
this.pullStatusInterval =
Integer.parseInt(
property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + ""));
this.maxLogLines = Integer.parseInt(property.getProperty("zeppelin.livy.maxLogLines", "1000"));
this.restTemplate = createRestTemplate();
if (!StringUtils.isBlank(property.getProperty("zeppelin.livy.http.headers"))) {
String[] headers = property.getProperty("zeppelin.livy.http.headers").split(";");
for (String header : headers) {
String[] splits = header.split(":", -1);
if (splits.length != 2) {
throw new RuntimeException("Invalid format of http headers: " + header +
", valid http header format is HEADER_NAME:HEADER_VALUE");
throw new RuntimeException(
"Invalid format of http headers: "
+ header
+ ", valid http header format is HEADER_NAME:HEADER_VALUE");
}
customHeaders.put(splits[0].trim(), envSubstitute(splits[1].trim()));
}
@ -163,8 +159,8 @@ public abstract class BaseLivyInterpreter extends Interpreter {
initLivySession();
}
} catch (LivyException e) {
String msg = "Fail to create session, please check livy interpreter log and " +
"livy server log";
String msg =
"Fail to create session, please check livy interpreter log and " + "livy server log";
throw new InterpreterException(msg, e);
}
}
@ -191,14 +187,17 @@ public abstract class BaseLivyInterpreter extends Interpreter {
sessionInfo.appId = extractAppId();
}
if (sessionInfo.appInfo == null ||
StringUtils.isEmpty(sessionInfo.appInfo.get("sparkUiUrl"))) {
if (sessionInfo.appInfo == null
|| StringUtils.isEmpty(sessionInfo.appInfo.get("sparkUiUrl"))) {
sessionInfo.webUIAddress = extractWebUIAddress();
} else {
sessionInfo.webUIAddress = sessionInfo.appInfo.get("sparkUiUrl");
}
LOGGER.info("Create livy session successfully with sessionId: {}, appId: {}, webUI: {}",
sessionInfo.id, sessionInfo.appId, sessionInfo.webUIAddress);
LOGGER.info(
"Create livy session successfully with sessionId: {}, appId: {}, webUI: {}",
sessionInfo.id,
sessionInfo.appId,
sessionInfo.webUIAddress);
} else {
LOGGER.info("Create livy session successfully with sessionId: {}", this.sessionInfo.id);
}
@ -235,20 +234,20 @@ public abstract class BaseLivyInterpreter extends Interpreter {
return interpret(st, null, context.getParagraphId(), this.displayAppInfo, true, true);
} catch (LivyException e) {
LOGGER.error("Fail to interpret:" + st, e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
return new InterpreterResult(
InterpreterResult.Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
List<InterpreterCompletion> candidates = Collections.emptyList();
try {
candidates = callCompletion(new CompletionRequest(buf, getSessionKind(), cursor));
} catch (SessionNotFoundException e) {
LOGGER.warn("Livy session {} is expired. Will return empty list of candidates.",
getSessionInfo().id);
LOGGER.warn(
"Livy session {} is expired. Will return empty list of candidates.", getSessionInfo().id);
} catch (LivyException le) {
logger.error("Failed to call code completions. Will return empty list of candidates", le);
}
@ -258,8 +257,10 @@ public abstract class BaseLivyInterpreter extends Interpreter {
private List<InterpreterCompletion> callCompletion(CompletionRequest req) throws LivyException {
List<InterpreterCompletion> candidates = new ArrayList<>();
try {
CompletionResponse resp = CompletionResponse.fromJson(
callRestAPI("/sessions/" + getSessionInfo().id + "/completion", "POST", req.toJson()));
CompletionResponse resp =
CompletionResponse.fromJson(
callRestAPI(
"/sessions/" + getSessionInfo().id + "/completion", "POST", req.toJson()));
for (String candidate : resp.candidates) {
candidates.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY));
}
@ -298,37 +299,51 @@ public abstract class BaseLivyInterpreter extends Interpreter {
return 0;
}
private SessionInfo createSession(String user, String kind)
throws LivyException {
private SessionInfo createSession(String user, String kind) throws LivyException {
try {
Map<String, String> conf = new HashMap<>();
for (Map.Entry<Object, Object> entry : getProperties().entrySet()) {
if (entry.getKey().toString().startsWith("livy.spark.") &&
!entry.getValue().toString().isEmpty()) {
if (entry.getKey().toString().startsWith("livy.spark.")
&& !entry.getValue().toString().isEmpty()) {
conf.put(entry.getKey().toString().substring(5), entry.getValue().toString());
}
}
CreateSessionRequest request = new CreateSessionRequest(kind,
user == null || user.equals("anonymous") ? null : user, conf);
SessionInfo sessionInfo = SessionInfo.fromJson(
callRestAPI("/sessions", "POST", request.toJson()));
CreateSessionRequest request =
new CreateSessionRequest(
kind, user == null || user.equals("anonymous") ? null : user, conf);
SessionInfo sessionInfo =
SessionInfo.fromJson(callRestAPI("/sessions", "POST", request.toJson()));
long start = System.currentTimeMillis();
// pull the session status until it is idle or timeout
while (!sessionInfo.isReady()) {
if ((System.currentTimeMillis() - start) / 1000 > sessionCreationTimeout) {
String msg = "The creation of session " + sessionInfo.id + " is timeout within "
+ sessionCreationTimeout + " seconds, appId: " + sessionInfo.appId
+ ", log:\n" + StringUtils.join(getSessionLog(sessionInfo.id).log, "\n");
String msg =
"The creation of session "
+ sessionInfo.id
+ " is timeout within "
+ sessionCreationTimeout
+ " seconds, appId: "
+ sessionInfo.appId
+ ", log:\n"
+ StringUtils.join(getSessionLog(sessionInfo.id).log, "\n");
throw new LivyException(msg);
}
Thread.sleep(pullStatusInterval);
sessionInfo = getSessionInfo(sessionInfo.id);
LOGGER.info("Session {} is in state {}, appId {}", sessionInfo.id, sessionInfo.state,
LOGGER.info(
"Session {} is in state {}, appId {}",
sessionInfo.id,
sessionInfo.state,
sessionInfo.appId);
if (sessionInfo.isFinished()) {
String msg = "Session " + sessionInfo.id + " is finished, appId: " + sessionInfo.appId
+ ", log:\n" + StringUtils.join(getSessionLog(sessionInfo.id).log, "\n");
String msg =
"Session "
+ sessionInfo.id
+ " is finished, appId: "
+ sessionInfo.appId
+ ", log:\n"
+ StringUtils.join(getSessionLog(sessionInfo.id).log, "\n");
throw new LivyException(msg);
}
}
@ -344,25 +359,34 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
private SessionLog getSessionLog(int sessionId) throws LivyException {
return SessionLog.fromJson(callRestAPI("/sessions/" + sessionId + "/log?size=" + maxLogLines,
"GET"));
return SessionLog.fromJson(
callRestAPI("/sessions/" + sessionId + "/log?size=" + maxLogLines, "GET"));
}
public InterpreterResult interpret(String code,
String paragraphId,
boolean displayAppInfo,
boolean appendSessionExpired,
boolean appendSessionDead) throws LivyException {
return interpret(code, sharedInterpreter.isSupported() ? getSessionKind() : null,
paragraphId, displayAppInfo, appendSessionExpired, appendSessionDead);
public InterpreterResult interpret(
String code,
String paragraphId,
boolean displayAppInfo,
boolean appendSessionExpired,
boolean appendSessionDead)
throws LivyException {
return interpret(
code,
sharedInterpreter.isSupported() ? getSessionKind() : null,
paragraphId,
displayAppInfo,
appendSessionExpired,
appendSessionDead);
}
public InterpreterResult interpret(String code,
String codeType,
String paragraphId,
boolean displayAppInfo,
boolean appendSessionExpired,
boolean appendSessionDead) throws LivyException {
public InterpreterResult interpret(
String code,
String codeType,
String paragraphId,
boolean displayAppInfo,
boolean appendSessionExpired,
boolean appendSessionDead)
throws LivyException {
StatementInfo stmtInfo = null;
boolean sessionExpired = false;
boolean sessionDead = false;
@ -393,8 +417,9 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
stmtInfo = executeStatement(new ExecuteRequest(code, codeType));
} else {
throw new LivyException("%html <font color=\"red\">Livy session is dead somehow, " +
"please check log to see why it is dead, and then restart livy interpreter</font>");
throw new LivyException(
"%html <font color=\"red\">Livy session is dead somehow, "
+ "please check log to see why it is dead, and then restart livy interpreter</font>");
}
}
@ -416,8 +441,8 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
}
if (appendSessionExpired || appendSessionDead) {
return appendSessionExpireDead(getResultFromStatementInfo(stmtInfo, displayAppInfo),
sessionExpired, sessionDead);
return appendSessionExpireDead(
getResultFromStatementInfo(stmtInfo, displayAppInfo), sessionExpired, sessionDead);
} else {
return getResultFromStatementInfo(stmtInfo, displayAppInfo);
}
@ -460,20 +485,20 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
}
private InterpreterResult appendSessionExpireDead(InterpreterResult result,
boolean sessionExpired,
boolean sessionDead) {
private InterpreterResult appendSessionExpireDead(
InterpreterResult result, boolean sessionExpired, boolean sessionDead) {
InterpreterResult result2 = new InterpreterResult(result.code());
if (sessionExpired) {
result2.add(InterpreterResult.Type.HTML,
"<font color=\"red\">Previous livy session is expired, new livy session is created. " +
"Paragraphs that depend on this paragraph need to be re-executed!</font>");
result2.add(
InterpreterResult.Type.HTML,
"<font color=\"red\">Previous livy session is expired, new livy session is created. "
+ "Paragraphs that depend on this paragraph need to be re-executed!</font>");
}
if (sessionDead) {
result2.add(InterpreterResult.Type.HTML,
"<font color=\"red\">Previous livy session is dead, new livy session is created. " +
"Paragraphs that depend on this paragraph need to be re-executed!</font>");
result2.add(
InterpreterResult.Type.HTML,
"<font color=\"red\">Previous livy session is dead, new livy session is created. "
+ "Paragraphs that depend on this paragraph need to be re-executed!</font>");
}
for (InterpreterResultMessage message : result.message()) {
@ -482,8 +507,8 @@ public abstract class BaseLivyInterpreter extends Interpreter {
return result2;
}
private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo,
boolean displayAppInfo) {
private InterpreterResult getResultFromStatementInfo(
StatementInfo stmtInfo, boolean displayAppInfo) {
if (stmtInfo.output != null && stmtInfo.output.isError()) {
InterpreterResult result = new InterpreterResult(InterpreterResult.Code.ERROR);
StringBuilder sb = new StringBuilder();
@ -505,7 +530,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
// This case should never happen, just in case
return new InterpreterResult(InterpreterResult.Code.ERROR, "Empty output");
} else {
//TODO(zjffdu) support other types of data (like json, image and etc)
// TODO(zjffdu) support other types of data (like json, image and etc)
String result = stmtInfo.output.data.plainText;
// check table magic result first
@ -526,11 +551,13 @@ public abstract class BaseLivyInterpreter extends Interpreter {
outputBuilder.append(StringUtils.join(row, "\t"));
outputBuilder.append("\n");
}
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.TABLE, outputBuilder.toString());
return new InterpreterResult(
InterpreterResult.Code.SUCCESS, InterpreterResult.Type.TABLE, outputBuilder.toString());
} else if (stmtInfo.output.data.imagePng != null) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.IMG, (String) stmtInfo.output.data.imagePng);
return new InterpreterResult(
InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.IMG,
(String) stmtInfo.output.data.imagePng);
} else if (result != null) {
result = result.trim();
if (result.startsWith("<link")
@ -544,9 +571,15 @@ public abstract class BaseLivyInterpreter extends Interpreter {
if (displayAppInfo) {
InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
interpreterResult.add(result);
String appInfoHtml = "<hr/>Spark Application Id: " + sessionInfo.appId + "<br/>"
+ "Spark WebUI: <a href=\"" + sessionInfo.webUIAddress + "\">"
+ sessionInfo.webUIAddress + "</a>";
String appInfoHtml =
"<hr/>Spark Application Id: "
+ sessionInfo.appId
+ "<br/>"
+ "Spark WebUI: <a href=\""
+ sessionInfo.webUIAddress
+ "\">"
+ sessionInfo.webUIAddress
+ "</a>";
interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml);
return interpreterResult;
} else {
@ -555,14 +588,13 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
}
private StatementInfo executeStatement(ExecuteRequest executeRequest)
throws LivyException {
return StatementInfo.fromJson(callRestAPI("/sessions/" + sessionInfo.id + "/statements", "POST",
executeRequest.toJson()));
private StatementInfo executeStatement(ExecuteRequest executeRequest) throws LivyException {
return StatementInfo.fromJson(
callRestAPI(
"/sessions/" + sessionInfo.id + "/statements", "POST", executeRequest.toJson()));
}
private StatementInfo getStatementInfo(int statementId)
throws LivyException {
private StatementInfo getStatementInfo(int statementId) throws LivyException {
return StatementInfo.fromJson(
callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId, "GET"));
}
@ -571,12 +603,11 @@ public abstract class BaseLivyInterpreter extends Interpreter {
callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId + "/cancel", "POST");
}
private RestTemplate createRestTemplate() {
String keytabLocation = getProperty("zeppelin.livy.keytab");
String principal = getProperty("zeppelin.livy.principal");
boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) &&
StringUtils.isNotEmpty(principal);
boolean isSpnegoEnabled =
StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal);
HttpClient httpClient = null;
if (livyURL.startsWith("https:")) {
@ -586,37 +617,37 @@ public abstract class BaseLivyInterpreter extends Interpreter {
throw new RuntimeException("No zeppelin.livy.ssl.trustStore specified for livy ssl");
}
if (StringUtils.isBlank(password)) {
throw new RuntimeException("No zeppelin.livy.ssl.trustStorePassword specified " +
"for livy ssl");
throw new RuntimeException(
"No zeppelin.livy.ssl.trustStorePassword specified " + "for livy ssl");
}
FileInputStream inputStream = null;
try {
inputStream = new FileInputStream(keystoreFile);
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(new FileInputStream(keystoreFile), password.toCharArray());
SSLContext sslContext = SSLContexts.custom()
.loadTrustMaterial(trustStore)
.build();
SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(trustStore).build();
SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
HttpClientBuilder httpClientBuilder = HttpClients.custom().setSSLSocketFactory(csf);
RequestConfig reqConfig = new RequestConfig() {
@Override
public boolean isAuthenticationEnabled() {
return true;
}
};
RequestConfig reqConfig =
new RequestConfig() {
@Override
public boolean isAuthenticationEnabled() {
return true;
}
};
httpClientBuilder.setDefaultRequestConfig(reqConfig);
Credentials credentials = new Credentials() {
@Override
public String getPassword() {
return null;
}
Credentials credentials =
new Credentials() {
@Override
public String getPassword() {
return null;
}
@Override
public Principal getUserPrincipal() {
return null;
}
};
@Override
public Principal getUserPrincipal() {
return null;
}
};
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY, credentials);
httpClientBuilder.setDefaultCredentialsProvider(credsProvider);
@ -656,8 +687,9 @@ public abstract class BaseLivyInterpreter extends Interpreter {
restTemplate = new RestTemplate(new HttpComponentsClientHttpRequestFactory(httpClient));
}
}
restTemplate.getMessageConverters().add(0,
new StringHttpMessageConverter(Charset.forName("UTF-8")));
restTemplate
.getMessageConverters()
.add(0, new StringHttpMessageConverter(Charset.forName("UTF-8")));
return restTemplate;
}
@ -689,8 +721,10 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
} catch (HttpClientErrorException e) {
response = new ResponseEntity(e.getResponseBodyAsString(), e.getStatusCode());
LOGGER.error(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), e.getResponseBodyAsString()));
LOGGER.error(
String.format(
"Error with %s StatusCode: %s",
response.getStatusCode().value(), e.getResponseBodyAsString()));
} catch (RestClientException e) {
// Exception happens when kerberos is enabled.
if (e.getCause() instanceof HttpClientErrorException) {
@ -698,8 +732,10 @@ public abstract class BaseLivyInterpreter extends Interpreter {
if (cause.getResponseBodyAsString().matches(SESSION_NOT_FOUND_PATTERN)) {
throw new SessionNotFoundException(cause.getResponseBodyAsString());
}
throw new LivyException(cause.getResponseBodyAsString() + "\n"
+ ExceptionUtils.getFullStackTrace(ExceptionUtils.getRootCause(e)));
throw new LivyException(
cause.getResponseBodyAsString()
+ "\n"
+ ExceptionUtils.getFullStackTrace(ExceptionUtils.getRootCause(e)));
}
if (e instanceof HttpServerErrorException) {
HttpServerErrorException errorException = (HttpServerErrorException) e;
@ -714,25 +750,27 @@ public abstract class BaseLivyInterpreter extends Interpreter {
if (response == null) {
throw new LivyException("No http response returned");
}
LOGGER.debug("Get response, StatusCode: {}, responseBody: {}", response.getStatusCode(),
LOGGER.debug(
"Get response, StatusCode: {}, responseBody: {}",
response.getStatusCode(),
response.getBody());
if (response.getStatusCode().value() == 200
|| response.getStatusCode().value() == 201) {
if (response.getStatusCode().value() == 200 || response.getStatusCode().value() == 201) {
return response.getBody();
} else if (response.getStatusCode().value() == 404) {
if (response.getBody().matches(SESSION_NOT_FOUND_PATTERN)) {
throw new SessionNotFoundException(response.getBody());
} else {
throw new APINotFoundException("No rest api found for " + targetURL +
", " + response.getStatusCode());
throw new APINotFoundException(
"No rest api found for " + targetURL + ", " + response.getStatusCode());
}
} else {
String responseString = response.getBody();
if (responseString.contains("CreateInteractiveRequest[\\\"master\\\"]")) {
return responseString;
}
throw new LivyException(String.format("Error with %s StatusCode: %s",
response.getStatusCode().value(), responseString));
throw new LivyException(
String.format(
"Error with %s StatusCode: %s", response.getStatusCode().value(), responseString));
}
}
@ -740,21 +778,23 @@ public abstract class BaseLivyInterpreter extends Interpreter {
try {
callRestAPI("/sessions/" + sessionId, "DELETE");
} catch (Exception e) {
LOGGER.error(String.format("Error closing session for user with session ID: %s",
sessionId), e);
LOGGER.error(
String.format("Error closing session for user with session ID: %s", sessionId), e);
}
}
/*
* We create these POJO here to accommodate livy 0.3 which is not released yet. livy rest api has
* some changes from version to version. So we create these POJO in zeppelin side to accommodate
* incompatibility between versions. Later, when livy become more stable, we could just depend on
* livy client jar.
*/
* We create these POJO here to accommodate livy 0.3 which is not released yet. livy rest api has
* some changes from version to version. So we create these POJO in zeppelin side to accommodate
* incompatibility between versions. Later, when livy become more stable, we could just depend on
* livy client jar.
*/
private static class CreateSessionRequest {
public final String kind;
@SerializedName("proxyUser")
public final String user;
public final Map<String, String> conf;
CreateSessionRequest(String kind, String user, Map<String, String> conf) {
@ -768,9 +808,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
}
/**
*
*/
/** */
public static class SessionInfo {
public final int id;
@ -783,8 +821,15 @@ public abstract class BaseLivyInterpreter extends Interpreter {
public final Map<String, String> appInfo;
public final List<String> log;
public SessionInfo(int id, String appId, String owner, String proxyUser, String state,
String kind, Map<String, String> appInfo, List<String> log) {
public SessionInfo(
int id,
String appId,
String owner,
String proxyUser,
String state,
String kind,
Map<String, String> appInfo,
List<String> log) {
this.id = id;
this.appId = appId;
this.owner = owner;
@ -814,8 +859,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
public int size;
public List<String> log;
SessionLog() {
}
SessionLog() {}
public static SessionLog fromJson(String json) {
return gson.fromJson(json, SessionLog.class);
@ -842,8 +886,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
public double progress;
public StatementOutput output;
StatementInfo() {
}
StatementInfo() {}
public static StatementInfo fromJson(String json) {
String rightJson = "";
@ -888,10 +931,13 @@ public abstract class BaseLivyInterpreter extends Interpreter {
private static class Data {
@SerializedName("text/plain")
public String plainText;
@SerializedName("image/png")
public String imagePng;
@SerializedName("application/json")
public String applicationJson;
@SerializedName("application/vnd.livy.table.v1+json")
public TableMagic applicationLivyTableJson;
}

11
livy/src/main/java/org/apache/zeppelin/livy/LivyException.java
View file

@ -19,12 +19,9 @@ package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.InterpreterException;
/**
* Livy api related exception.
*/
/** Livy api related exception. */
public class LivyException extends InterpreterException {
public LivyException() {
}
public LivyException() {}
public LivyException(String message) {
super(message);
@ -38,8 +35,8 @@ public class LivyException extends InterpreterException {
super(cause);
}
public LivyException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
public LivyException(
String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

6
livy/src/main/java/org/apache/zeppelin/livy/LivyPySpark3Interpreter.java
View file

@ -19,10 +19,7 @@ package org.apache.zeppelin.livy;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
/** Livy PySpark interpreter for Zeppelin. */
public class LivyPySpark3Interpreter extends LivyPySparkBaseInterpreter {
public LivyPySpark3Interpreter(Properties property) {
@ -33,5 +30,4 @@ public class LivyPySpark3Interpreter extends LivyPySparkBaseInterpreter {
public String getSessionKind() {
return "pyspark3";
}
}

23
livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkBaseInterpreter.java
View file

@ -19,9 +19,7 @@ package org.apache.zeppelin.livy;
import java.util.Properties;
/**
* Base class for PySpark Interpreter.
*/
/** Base class for PySpark Interpreter. */
public abstract class LivyPySparkBaseInterpreter extends BaseLivyInterpreter {
public LivyPySparkBaseInterpreter(Properties property) {
@ -31,22 +29,21 @@ public abstract class LivyPySparkBaseInterpreter extends BaseLivyInterpreter {
@Override
protected String extractAppId() throws LivyException {
return extractStatementResult(
interpret("sc.applicationId", null, false, false, false).message()
.get(0).getData());
interpret("sc.applicationId", null, false, false, false).message().get(0).getData());
}
@Override
protected String extractWebUIAddress() throws LivyException {
return extractStatementResult(
interpret(
"sc._jsc.sc().ui().get().appUIAddress()", null, false, false, false)
.message().get(0).getData());
interpret("sc._jsc.sc().ui().get().appUIAddress()", null, false, false, false)
.message()
.get(0)
.getData());
}
/**
* Extract the eval result of spark shell, e.g. extract application_1473129941656_0048
* from following:
* u'application_1473129941656_0048'
* Extract the eval result of spark shell, e.g. extract application_1473129941656_0048 from
* following: u'application_1473129941656_0048'
*
* @param result
* @return
@ -56,8 +53,8 @@ public abstract class LivyPySparkBaseInterpreter extends BaseLivyInterpreter {
if ((pos = result.indexOf("'")) >= 0) {
return result.substring(pos + 1, result.length() - 1).trim();
} else {
throw new RuntimeException("No result can be extracted from '" + result + "', " +
"something must be wrong");
throw new RuntimeException(
"No result can be extracted from '" + result + "', " + "something must be wrong");
}
}
}

7
livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java
View file

@ -19,10 +19,7 @@ package org.apache.zeppelin.livy;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
/** Livy PySpark interpreter for Zeppelin. */
public class LivyPySparkInterpreter extends LivyPySparkBaseInterpreter {
public LivyPySparkInterpreter(Properties property) {
@ -33,6 +30,4 @@ public class LivyPySparkInterpreter extends LivyPySparkBaseInterpreter {
public String getSessionKind() {
return "pyspark";
}
}

20
livy/src/main/java/org/apache/zeppelin/livy/LivySharedInterpreter.java
View file

@ -17,20 +17,16 @@
package org.apache.zeppelin.livy;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Livy Interpreter for shared kind which share SparkContext across spark/pyspark/r.
*/
/** Livy Interpreter for shared kind which share SparkContext across spark/pyspark/r. */
public class LivySharedInterpreter extends BaseLivyInterpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(LivySharedInterpreter.class);
@ -63,8 +59,8 @@ public class LivySharedInterpreter extends BaseLivyInterpreter {
isSupported = false;
}
} catch (LivyException e) {
String msg = "Fail to create session, please check livy interpreter log and " +
"livy server log";
String msg =
"Fail to create session, please check livy interpreter log and " + "livy server log";
throw new InterpreterException(msg, e);
}
}
@ -82,8 +78,8 @@ public class LivySharedInterpreter extends BaseLivyInterpreter {
return interpret(st, codeType, context.getParagraphId(), this.displayAppInfo, true, true);
} catch (LivyException e) {
LOGGER.error("Fail to interpret:" + st, e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
return new InterpreterResult(
InterpreterResult.Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
}

31
livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java
View file

@ -19,9 +19,7 @@ package org.apache.zeppelin.livy;
import java.util.Properties;
/**
* Livy Spark interpreter for Zeppelin.
*/
/** Livy Spark interpreter for Zeppelin. */
public class LivySparkInterpreter extends BaseLivyInterpreter {
public LivySparkInterpreter(Properties property) {
@ -36,8 +34,7 @@ public class LivySparkInterpreter extends BaseLivyInterpreter {
@Override
protected String extractAppId() throws LivyException {
return extractStatementResult(
interpret("sc.applicationId", null, false, false, false).message()
.get(0).getData());
interpret("sc.applicationId", null, false, false, false).message().get(0).getData());
}
@Override
@ -45,17 +42,25 @@ public class LivySparkInterpreter extends BaseLivyInterpreter {
interpret(
"val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get",
null,
null, false, false, false);
null,
false,
false,
false);
return extractStatementResult(
interpret(
"webui.getClass.getMethod(\"appUIAddress\").invoke(webui)", null, false, false, false)
.message().get(0).getData());
"webui.getClass.getMethod(\"appUIAddress\").invoke(webui)",
null,
false,
false,
false)
.message()
.get(0)
.getData());
}
/**
* Extract the eval result of spark shell, e.g. extract application_1473129941656_0048
* from following:
* res0: String = application_1473129941656_0048
* Extract the eval result of spark shell, e.g. extract application_1473129941656_0048 from
* following: res0: String = application_1473129941656_0048
*
* @param result
* @return
@ -65,8 +70,8 @@ public class LivySparkInterpreter extends BaseLivyInterpreter {
if ((pos = result.indexOf("=")) >= 0) {
return result.substring(pos + 1).trim();
} else {
throw new RuntimeException("No result can be extracted from '" + result + "', " +
"something must be wrong");
throw new RuntimeException(
"No result can be extracted from '" + result + "', " + "something must be wrong");
}
}
}

9
livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java
View file

@ -19,10 +19,7 @@ package org.apache.zeppelin.livy;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
/** Livy PySpark interpreter for Zeppelin. */
public class LivySparkRInterpreter extends BaseLivyInterpreter {
public LivySparkRInterpreter(Properties property) {
@ -36,13 +33,13 @@ public class LivySparkRInterpreter extends BaseLivyInterpreter {
@Override
protected String extractAppId() throws LivyException {
//TODO(zjffdu) depends on SparkR
// TODO(zjffdu) depends on SparkR
return null;
}
@Override
protected String extractWebUIAddress() throws LivyException {
//TODO(zjffdu) depends on SparkR
// TODO(zjffdu) depends on SparkR
return null;
}
}

64
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
View file

@ -17,6 +17,11 @@
package org.apache.zeppelin.livy;
import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
@ -28,15 +33,7 @@ import org.apache.zeppelin.interpreter.ResultMessages;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import static org.apache.commons.lang.StringEscapeUtils.escapeJavaScript;
/**
* Livy SparkSQL Interpreter for Zeppelin.
*/
/** Livy SparkSQL Interpreter for Zeppelin. */
public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
public static final String ZEPPELIN_LIVY_SPARK_SQL_FIELD_TRUNCATE =
"zeppelin.livy.spark.sql.field.truncate";
@ -71,13 +68,13 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
// As we don't know whether livyserver use spark2 or spark1, so we will detect SparkSession
// to judge whether it is using spark2.
try {
InterpreterContext context = InterpreterContext.builder()
.setInterpreterOut(new InterpreterOutput(null))
.build();
InterpreterContext context =
InterpreterContext.builder().setInterpreterOut(new InterpreterOutput(null)).build();
InterpreterResult result = sparkInterpreter.interpret("spark", context);
if (result.code() == InterpreterResult.Code.SUCCESS &&
result.message().get(0).getData().contains("org.apache.spark.sql.SparkSession")) {
LOGGER.info("SparkSession is detected so we are using spark 2.x for session {}",
if (result.code() == InterpreterResult.Code.SUCCESS
&& result.message().get(0).getData().contains("org.apache.spark.sql.SparkSession")) {
LOGGER.info(
"SparkSession is detected so we are using spark 2.x for session {}",
sparkInterpreter.getSessionInfo().id);
isSpark2 = true;
} else {
@ -89,12 +86,14 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
// create SqlContext if it is not available, as in livy 0.2 sqlContext
// is not available.
LOGGER.info("sqlContext is not detected, try to create SQLContext by ourselves");
result = sparkInterpreter.interpret(
"val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
+ "import sqlContext.implicits._", context);
result =
sparkInterpreter.interpret(
"val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n"
+ "import sqlContext.implicits._",
context);
if (result.code() == InterpreterResult.Code.ERROR) {
throw new LivyException("Fail to create SQLContext," +
result.message().get(0).getData());
throw new LivyException(
"Fail to create SQLContext," + result.message().get(0).getData());
}
}
}
@ -113,11 +112,10 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
// use triple quote so that we don't need to do string escape.
String sqlQuery = null;
if (isSpark2) {
sqlQuery = "spark.sql(\"\"\"" + line + "\"\"\").show(" + maxResult + ", " +
truncate + ")";
sqlQuery = "spark.sql(\"\"\"" + line + "\"\"\").show(" + maxResult + ", " + truncate + ")";
} else {
sqlQuery = "sqlContext.sql(\"\"\"" + line + "\"\"\").show(" + maxResult + ", " +
truncate + ")";
sqlQuery =
"sqlContext.sql(\"\"\"" + line + "\"\"\").show(" + maxResult + ", " + truncate + ")";
}
InterpreterResult result = sparkInterpreter.interpret(sqlQuery, context);
if (result.code() == InterpreterResult.Code.SUCCESS) {
@ -130,8 +128,9 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
List<String> rows = parseSQLOutput(message.getData());
result2.add(InterpreterResult.Type.TABLE, StringUtils.join(rows, "\n"));
if (rows.size() >= (maxResult + 1)) {
result2.add(ResultMessages.getExceedsLimitRowsMessage(maxResult,
ZEPPELIN_LIVY_SPARK_SQL_MAX_RESULT));
result2.add(
ResultMessages.getExceedsLimitRowsMessage(
maxResult, ZEPPELIN_LIVY_SPARK_SQL_MAX_RESULT));
}
} else {
result2.add(message.getType(), message.getData());
@ -143,8 +142,8 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
}
} catch (Exception e) {
LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
return new InterpreterResult(
InterpreterResult.Code.ERROR, InterpreterUtils.getMostRelevantMessage(e));
}
}
@ -202,9 +201,7 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
return rows;
}
/**
* Represent the start and end index of each cell.
*/
/** Represent the start and end index of each cell. */
private static class Pair {
private int start;
private int end;
@ -223,8 +220,9 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter {
public Scheduler getScheduler() {
if (concurrentSQL()) {
int maxConcurrency = 10;
return SchedulerFactory.singleton().createOrGetParallelScheduler(
LivySparkInterpreter.class.getName() + this.hashCode(), maxConcurrency);
return SchedulerFactory.singleton()
.createOrGetParallelScheduler(
LivySparkInterpreter.class.getName() + this.hashCode(), maxConcurrency);
} else {
if (sparkInterpreter != null) {
return sparkInterpreter.getScheduler();

8
livy/src/main/java/org/apache/zeppelin/livy/LivyVersion.java
View file

@ -20,9 +20,7 @@ package org.apache.zeppelin.livy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Provide reading comparing capability of livy version.
*/
/** Provide reading comparing capability of livy version. */
public class LivyVersion {
private static final Logger logger = LoggerFactory.getLogger(LivyVersion.class);
@ -52,8 +50,8 @@ public class LivyVersion {
// version is always 5 digits. (e.g. 2.0.0 -> 20000, 1.6.2 -> 10602)
version = Integer.parseInt(String.format("%d%02d%02d", major, minor, patch));
} catch (Exception e) {
logger.error("Can not recognize Livy version " + versionString +
". Assume it's a future release", e);
logger.error(
"Can not recognize Livy version " + versionString + ". Assume it's a future release", e);
// assume it is future release
version = 99999;

4
livy/src/main/java/org/apache/zeppelin/livy/SessionDeadException.java
View file

@ -17,6 +17,4 @@
package org.apache.zeppelin.livy;
public class SessionDeadException extends LivyException {
}
public class SessionDeadException extends LivyException {}

4
livy/src/main/java/org/apache/zeppelin/livy/SessionNotFoundException.java
View file

@ -17,9 +17,7 @@
package org.apache.zeppelin.livy;
/**
*
*/
/** */
public class SessionNotFoundException extends LivyException {
public SessionNotFoundException(String message) {

574
livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
View file

@ -17,6 +17,14 @@
package org.apache.zeppelin.livy;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.apache.commons.io.IOUtils;
import org.apache.livy.test.framework.Cluster;
import org.apache.livy.test.framework.Cluster$;
@ -37,15 +45,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
public class LivyInterpreterIT {
private static final Logger LOGGER = LoggerFactory.getLogger(LivyInterpreterIT.class);
private static Cluster cluster;
@ -85,7 +84,6 @@ public class LivyInterpreterIT {
return true;
}
@Test
public void testSparkInterpreter() throws InterpreterException {
if (!checkPreCondition()) {
@ -99,12 +97,13 @@ public class LivyInterpreterIT {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
InterpreterContext context =
InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
sparkInterpreter.open();
LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
@ -132,12 +131,13 @@ public class LivyInterpreterIT {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
final InterpreterContext context = InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
final InterpreterContext context =
InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
;
InterpreterResult result = sparkInterpreter.interpret("sc.parallelize(1 to 10).sum()", context);
@ -158,18 +158,15 @@ public class LivyInterpreterIT {
assertEquals(1, result.message().size());
// multi-line string
String multiLineString = "val str = \"\"\"multiple\n" +
"line\"\"\"\n" +
"println(str)";
String multiLineString = "val str = \"\"\"multiple\n" + "line\"\"\"\n" + "println(str)";
result = sparkInterpreter.interpret(multiLineString, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("multiple\nline"));
// case class
String caseClassCode = "case class Person(id:Int, \n" +
"name:String)\n" +
"val p=Person(1, \"name_a\")";
String caseClassCode =
"case class Person(id:Int, \n" + "name:String)\n" + "val p=Person(1, \"name_a\")";
result = sparkInterpreter.interpret(caseClassCode, context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
@ -207,60 +204,78 @@ public class LivyInterpreterIT {
// cancel
if (sparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) {
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
sparkInterpreter.cancel(context);
}
};
Thread cancelThread =
new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
sparkInterpreter.cancel(context);
}
};
cancelThread.start();
result = sparkInterpreter
.interpret("sc.parallelize(1 to 10).foreach(e=>Thread.sleep(10*1000))", context);
result =
sparkInterpreter.interpret(
"sc.parallelize(1 to 10).foreach(e=>Thread.sleep(10*1000))", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") ||
message.contains("Job is cancelled"));
assertTrue(
message.contains("cancelled part of cancelled job group")
|| message.contains("Job is cancelled"));
}
}
private void testDataFrame(LivySparkInterpreter sparkInterpreter,
final LivySparkSQLInterpreter sqlInterpreter,
boolean isSpark2) throws LivyException {
private void testDataFrame(
LivySparkInterpreter sparkInterpreter,
final LivySparkSQLInterpreter sqlInterpreter,
boolean isSpark2)
throws LivyException {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
final InterpreterContext context = InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
final InterpreterContext context =
InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
InterpreterResult result = null;
// test DataFrame api
if (!isSpark2) {
result = sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
result =
sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
} else {
result = sparkInterpreter.interpret(
"val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
result =
sparkInterpreter.interpret(
"val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
}
sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
// test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter
@ -299,57 +314,70 @@ public class LivyInterpreterIT {
// test sql cancel
if (sqlInterpreter.getLivyVersion().newerThanEquals(LivyVersion.LIVY_0_3_0)) {
Thread cancelThread = new Thread() {
@Override
public void run() {
sqlInterpreter.cancel(context);
}
};
Thread cancelThread =
new Thread() {
@Override
public void run() {
sqlInterpreter.cancel(context);
}
};
cancelThread.start();
//sleep so that cancelThread performs a cancel.
// sleep so that cancelThread performs a cancel.
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
result = sqlInterpreter
.interpret("select count(1) from df", context);
result = sqlInterpreter.interpret("select count(1) from df", context);
if (result.code().equals(InterpreterResult.Code.ERROR)) {
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") ||
message.contains("Job is cancelled"));
assertTrue(
message.contains("cancelled part of cancelled job group")
|| message.contains("Job is cancelled"));
}
}
// test result string truncate
if (!isSpark2) {
result = sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20)))"
+ ".toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
result =
sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20)))"
+ ".toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])"));
} else {
result = sparkInterpreter.interpret(
"val df=spark.createDataFrame(Seq((\"12characters12characters\",20)))"
+ ".toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
result =
sparkInterpreter.interpret(
"val df=spark.createDataFrame(Seq((\"12characters12characters\",20)))"
+ ".toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])"));
}
sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
// test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter
result = sqlInterpreter.interpret("select * from df where col_1='12characters12characters'",
context);
result =
sqlInterpreter.interpret(
"select * from df where col_1='12characters12characters'", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
assertEquals("col_1\tcol_2\n12characters12cha...\t20", result.message().get(0).getData());
}
@Test
@ -363,12 +391,13 @@ public class LivyInterpreterIT {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
final InterpreterContext context = InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
final InterpreterContext context =
InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
pysparkInterpreter.open();
// test traceback msg
@ -376,8 +405,8 @@ public class LivyInterpreterIT {
pysparkInterpreter.getLivyVersion();
// for livy version >=0.3 , input some erroneous spark code, check the shown result is more
// than one line
InterpreterResult result = pysparkInterpreter.interpret(
"sc.parallelize(wrongSyntax(1, 2)).count()", context);
InterpreterResult result =
pysparkInterpreter.interpret("sc.parallelize(wrongSyntax(1, 2)).count()", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertTrue(result.message().get(0).getData().split("\n").length > 1);
assertTrue(result.message().get(0).getData().contains("Traceback"));
@ -397,7 +426,7 @@ public class LivyInterpreterIT {
assertEquals(InterpreterResult.Code.SUCCESS, reslt.code());
assertTrue(reslt.message().get(0).getData().contains(utf8Str));
//test special characters
// test special characters
String charStr = "açñiñíûÑoç";
InterpreterResult res = pysparkInterpreter.interpret("print(\"" + charStr + "\")", context);
assertEquals(InterpreterResult.Code.SUCCESS, res.code());
@ -418,28 +447,34 @@ public class LivyInterpreterIT {
// test DataFrame api
if (!isSpark2) {
pysparkInterpreter.interpret("from pyspark.sql import SQLContext\n"
+ "sqlContext = SQLContext(sc)", context);
result = pysparkInterpreter.interpret("df=sqlContext.createDataFrame([(\"hello\",20)])\n"
+ "df.collect()", context);
pysparkInterpreter.interpret(
"from pyspark.sql import SQLContext\n" + "sqlContext = SQLContext(sc)", context);
result =
pysparkInterpreter.interpret(
"df=sqlContext.createDataFrame([(\"hello\",20)])\n" + "df.collect()", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
//python2 has u and python3 don't have u
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]")
|| result.message().get(0).getData().contains("[Row(_1='hello', _2=20)]"));
// python2 has u and python3 don't have u
assertTrue(
result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]")
|| result.message().get(0).getData().contains("[Row(_1='hello', _2=20)]"));
} else {
result = pysparkInterpreter.interpret("df=spark.createDataFrame([(\"hello\",20)])\n"
+ "df.collect()", context);
result =
pysparkInterpreter.interpret(
"df=spark.createDataFrame([(\"hello\",20)])\n" + "df.collect()", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
//python2 has u and python3 don't have u
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]")
|| result.message().get(0).getData().contains("[Row(_1='hello', _2=20)]"));
// python2 has u and python3 don't have u
assertTrue(
result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]")
|| result.message().get(0).getData().contains("[Row(_1='hello', _2=20)]"));
}
// test magic api
pysparkInterpreter.interpret("t = [{\"name\":\"userA\", \"role\":\"roleA\"},"
+ "{\"name\":\"userB\", \"role\":\"roleB\"}]", context);
pysparkInterpreter.interpret(
"t = [{\"name\":\"userA\", \"role\":\"roleA\"},"
+ "{\"name\":\"userB\", \"role\":\"roleB\"}]",
context);
result = pysparkInterpreter.interpret("%table t", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
@ -454,27 +489,29 @@ public class LivyInterpreterIT {
// cancel
if (pysparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) {
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
pysparkInterpreter.cancel(context);
}
};
Thread cancelThread =
new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
pysparkInterpreter.cancel(context);
}
};
cancelThread.start();
result = pysparkInterpreter
.interpret("import time\n" +
"sc.range(1, 10).foreach(lambda a: time.sleep(10))", context);
result =
pysparkInterpreter.interpret(
"import time\n" + "sc.range(1, 10).foreach(lambda a: time.sleep(10))", context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") ||
message.contains("Job is cancelled"));
assertTrue(
message.contains("cancelled part of cancelled job group")
|| message.contains("Job is cancelled"));
}
} finally {
pysparkInterpreter.close();
@ -500,12 +537,13 @@ public class LivyInterpreterIT {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
InterpreterContext context =
InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
sparkInterpreter.open();
LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties2);
@ -535,28 +573,43 @@ public class LivyInterpreterIT {
boolean isSpark2 = isSpark2(sparkInterpreter, context);
if (!isSpark2) {
result = sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20)))"
+ ".toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
result =
sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"12characters12characters\",20)))"
+ ".toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(2, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains(
"Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])"));
} else {
result = sparkInterpreter.interpret(
"val df=spark.createDataFrame(Seq((\"12characters12characters\",20)))"
+ ".toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
result =
sparkInterpreter.interpret(
"val df=spark.createDataFrame(Seq((\"12characters12characters\",20)))"
+ ".toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(2, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains(
"Array[org.apache.spark.sql.Row] = Array([12characters12characters,20])"));
}
sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
// test LivySparkSQLInterpreter which share the same SparkContext with LivySparkInterpreter
result = sqlInterpreter.interpret("select * from df where col_1='12characters12characters'",
context);
result =
sqlInterpreter.interpret(
"select * from df where col_1='12characters12characters'", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
assertEquals("col_1\tcol_2\n12characters12characters\t20", result.message().get(0).getData());
@ -584,12 +637,13 @@ public class LivyInterpreterIT {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
final InterpreterContext context = InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
final InterpreterContext context =
InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
sparkRInterpreter.open();
try {
@ -604,30 +658,36 @@ public class LivyInterpreterIT {
assertTrue(result.message().get(0).getData().contains("eruptions waiting"));
// cancel
Thread cancelThread = new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
sparkRInterpreter.cancel(context);
}
};
Thread cancelThread =
new Thread() {
@Override
public void run() {
// invoke cancel after 1 millisecond to wait job starting
try {
Thread.sleep(1);
} catch (InterruptedException e) {
e.printStackTrace();
}
sparkRInterpreter.cancel(context);
}
};
cancelThread.start();
result = sparkRInterpreter.interpret("df <- as.DataFrame(faithful)\n" +
"df1 <- dapplyCollect(df, function(x) " +
"{ Sys.sleep(10); x <- cbind(x, x$waiting * 60) })", context);
result =
sparkRInterpreter.interpret(
"df <- as.DataFrame(faithful)\n"
+ "df1 <- dapplyCollect(df, function(x) "
+ "{ Sys.sleep(10); x <- cbind(x, x$waiting * 60) })",
context);
assertEquals(InterpreterResult.Code.ERROR, result.code());
String message = result.message().get(0).getData();
// 2 possibilities, sometimes livy doesn't return the real cancel exception
assertTrue(message.contains("cancelled part of cancelled job group") ||
message.contains("Job is cancelled"));
assertTrue(
message.contains("cancelled part of cancelled job group")
|| message.contains("Job is cancelled"));
} else {
result = sparkRInterpreter.interpret("df <- createDataFrame(sqlContext, faithful)" +
"\nhead(df)", context);
result =
sparkRInterpreter.interpret(
"df <- createDataFrame(sqlContext, faithful)" + "\nhead(df)", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("eruptions waiting"));
@ -650,12 +710,12 @@ public class LivyInterpreterIT {
}
InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
interpreterGroup.put("session_1", new ArrayList<Interpreter>());
LazyOpenInterpreter sparkInterpreter = new LazyOpenInterpreter(
new LivySparkInterpreter(properties));
LazyOpenInterpreter sparkInterpreter =
new LazyOpenInterpreter(new LivySparkInterpreter(properties));
sparkInterpreter.setInterpreterGroup(interpreterGroup);
interpreterGroup.get("session_1").add(sparkInterpreter);
LazyOpenInterpreter sqlInterpreter = new LazyOpenInterpreter(
new LivySparkSQLInterpreter(properties));
LazyOpenInterpreter sqlInterpreter =
new LazyOpenInterpreter(new LivySparkSQLInterpreter(properties));
interpreterGroup.get("session_1").add(sqlInterpreter);
sqlInterpreter.setInterpreterGroup(interpreterGroup);
sqlInterpreter.open();
@ -664,12 +724,13 @@ public class LivyInterpreterIT {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
InterpreterContext context =
InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
String p1 = IOUtils.toString(getClass().getResourceAsStream("/livy_tutorial_1.scala"));
InterpreterResult result = sparkInterpreter.interpret(p1, context);
@ -692,28 +753,28 @@ public class LivyInterpreterIT {
}
InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
interpreterGroup.put("session_1", new ArrayList<Interpreter>());
LazyOpenInterpreter sparkInterpreter = new LazyOpenInterpreter(
new LivySparkInterpreter(properties));
LazyOpenInterpreter sparkInterpreter =
new LazyOpenInterpreter(new LivySparkInterpreter(properties));
sparkInterpreter.setInterpreterGroup(interpreterGroup);
interpreterGroup.get("session_1").add(sparkInterpreter);
LazyOpenInterpreter sqlInterpreter = new LazyOpenInterpreter(
new LivySparkSQLInterpreter(properties));
LazyOpenInterpreter sqlInterpreter =
new LazyOpenInterpreter(new LivySparkSQLInterpreter(properties));
interpreterGroup.get("session_1").add(sqlInterpreter);
sqlInterpreter.setInterpreterGroup(interpreterGroup);
LazyOpenInterpreter pysparkInterpreter = new LazyOpenInterpreter(
new LivyPySparkInterpreter(properties));
LazyOpenInterpreter pysparkInterpreter =
new LazyOpenInterpreter(new LivyPySparkInterpreter(properties));
interpreterGroup.get("session_1").add(pysparkInterpreter);
pysparkInterpreter.setInterpreterGroup(interpreterGroup);
LazyOpenInterpreter sparkRInterpreter = new LazyOpenInterpreter(
new LivySparkRInterpreter(properties));
LazyOpenInterpreter sparkRInterpreter =
new LazyOpenInterpreter(new LivySparkRInterpreter(properties));
interpreterGroup.get("session_1").add(sparkRInterpreter);
sparkRInterpreter.setInterpreterGroup(interpreterGroup);
LazyOpenInterpreter sharedInterpreter = new LazyOpenInterpreter(
new LivySharedInterpreter(properties));
LazyOpenInterpreter sharedInterpreter =
new LazyOpenInterpreter(new LivySharedInterpreter(properties));
interpreterGroup.get("session_1").add(sharedInterpreter);
sharedInterpreter.setInterpreterGroup(interpreterGroup);
@ -726,68 +787,91 @@ public class LivyInterpreterIT {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
InterpreterContext context =
InterpreterContext.builder()
.setNoteId("noteId")
.setParagraphId("paragraphId")
.setAuthenticationInfo(authInfo)
.setInterpreterOut(output)
.build();
// detect spark version
InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
boolean isSpark2 = isSpark2((BaseLivyInterpreter) sparkInterpreter.getInnerInterpreter(),
context);
boolean isSpark2 =
isSpark2((BaseLivyInterpreter) sparkInterpreter.getInnerInterpreter(), context);
if (!isSpark2) {
result = sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
result =
sparkInterpreter.interpret(
"val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
// access table from pyspark
result = pysparkInterpreter.interpret("sqlContext.sql(\"select * from df\").show()",
context);
result =
pysparkInterpreter.interpret("sqlContext.sql(\"select * from df\").show()", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("+-----+-----+\n" +
"|col_1|col_2|\n" +
"+-----+-----+\n" +
"|hello| 20|\n" +
"+-----+-----+"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains(
"+-----+-----+\n"
+ "|col_1|col_2|\n"
+ "+-----+-----+\n"
+ "|hello| 20|\n"
+ "+-----+-----+"));
// access table from sparkr
result = sparkRInterpreter.interpret("head(sql(sqlContext, \"select * from df\"))",
context);
result =
sparkRInterpreter.interpret("head(sql(sqlContext, \"select * from df\"))", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData().contains("col_1 col_2\n1 hello 20"));
} else {
result = sparkInterpreter.interpret(
"val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()", context);
result =
sparkInterpreter.interpret(
"val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
+ "df.collect()",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
// access table from pyspark
result = pysparkInterpreter.interpret("spark.sql(\"select * from df\").show()", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertTrue(result.message().get(0).getData()
.contains("+-----+-----+\n" +
"|col_1|col_2|\n" +
"+-----+-----+\n" +
"|hello| 20|\n" +
"+-----+-----+"));
assertTrue(
result
.message()
.get(0)
.getData()
.contains(
"+-----+-----+\n"
+ "|col_1|col_2|\n"
+ "+-----+-----+\n"
+ "|hello| 20|\n"
+ "+-----+-----+"));
// access table from sparkr
result = sparkRInterpreter.interpret("head(sql(\"select * from df\"))", context);
@ -797,27 +881,28 @@ public class LivyInterpreterIT {
}
// test plotting of python
result = pysparkInterpreter.interpret(
"import matplotlib.pyplot as plt\n" +
"plt.switch_backend('agg')\n" +
"data=[1,2,3,4]\n" +
"plt.figure()\n" +
"plt.plot(data)\n" +
"%matplot plt", context);
result =
pysparkInterpreter.interpret(
"import matplotlib.pyplot as plt\n"
+ "plt.switch_backend('agg')\n"
+ "data=[1,2,3,4]\n"
+ "plt.figure()\n"
+ "plt.plot(data)\n"
+ "%matplot plt",
context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertEquals(InterpreterResult.Type.IMG, result.message().get(0).getType());
// test plotting of R
result = sparkRInterpreter.interpret(
"hist(mtcars$mpg)", context);
result = sparkRInterpreter.interpret("hist(mtcars$mpg)", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(1, result.message().size());
assertEquals(InterpreterResult.Type.IMG, result.message().get(0).getType());
// test code completion
List<InterpreterCompletion> completionResult = sparkInterpreter
.completion("df.sho", 6, context);
List<InterpreterCompletion> completionResult =
sparkInterpreter.completion("df.sho", 6, context);
assertEquals(1, completionResult.size());
assertEquals("show", completionResult.get(0).name);
@ -849,17 +934,12 @@ public class LivyInterpreterIT {
public static class MyInterpreterOutputListener implements InterpreterOutputListener {
@Override
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
}
public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {}
@Override
public void onUpdate(int index, InterpreterResultMessageOutput out) {
}
public void onUpdate(int index, InterpreterResultMessageOutput out) {}
@Override
public void onUpdateAll(InterpreterOutput out) {
}
public void onUpdateAll(InterpreterOutput out) {}
}
}

93
livy/src/test/java/org/apache/zeppelin/livy/LivySQLInterpreterTest.java
View file

@ -21,15 +21,12 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import java.util.List;
import java.util.Properties;
import org.junit.Before;
import org.junit.Test;
import java.util.List;
import java.util.Properties;
/**
* Unit test for LivySQLInterpreter.
*/
/** Unit test for LivySQLInterpreter. */
public class LivySQLInterpreterTest {
private LivySparkSQLInterpreter sqlInterpreter;
@ -58,14 +55,11 @@ public class LivySQLInterpreterTest {
// | a| b|
// +---+---+
// +---+---+
List<String> rows = sqlInterpreter.parseSQLOutput("+---+---+\n" +
"| a| b|\n" +
"+---+---+\n" +
"+---+---+");
List<String> rows =
sqlInterpreter.parseSQLOutput("+---+---+\n" + "| a| b|\n" + "+---+---+\n" + "+---+---+");
assertEquals(1, rows.size());
assertEquals("a\tb", rows.get(0));
// sql output with 2 rows
// +---+---+
// | a| b|
@ -73,18 +67,19 @@ public class LivySQLInterpreterTest {
// | 1| 1a|
// | 2| 2b|
// +---+---+
rows = sqlInterpreter.parseSQLOutput("+---+---+\n" +
"| a| b|\n" +
"+---+---+\n" +
"| 1| 1a|\n" +
"| 2| 2b|\n" +
"+---+---+");
rows =
sqlInterpreter.parseSQLOutput(
"+---+---+\n"
+ "| a| b|\n"
+ "+---+---+\n"
+ "| 1| 1a|\n"
+ "| 2| 2b|\n"
+ "+---+---+");
assertEquals(3, rows.size());
assertEquals("a\tb", rows.get(0));
assertEquals("1\t1a", rows.get(1));
assertEquals("2\t2b", rows.get(2));
// sql output with 3 rows and showing "only showing top 3 rows"
// +---+---+
// | a| b|
@ -94,21 +89,22 @@ public class LivySQLInterpreterTest {
// | 3| 3c|
// +---+---+
// only showing top 3 rows
rows = sqlInterpreter.parseSQLOutput("+---+---+\n" +
"| a| b|\n" +
"+---+---+\n" +
"| 1| 1a|\n" +
"| 2| 2b|\n" +
"| 3| 3c|\n" +
"+---+---+\n" +
"only showing top 3 rows");
rows =
sqlInterpreter.parseSQLOutput(
"+---+---+\n"
+ "| a| b|\n"
+ "+---+---+\n"
+ "| 1| 1a|\n"
+ "| 2| 2b|\n"
+ "| 3| 3c|\n"
+ "+---+---+\n"
+ "only showing top 3 rows");
assertEquals(4, rows.size());
assertEquals("a\tb", rows.get(0));
assertEquals("1\t1a", rows.get(1));
assertEquals("2\t2b", rows.get(2));
assertEquals("3\t3c", rows.get(3));
// sql output with 1 rows and showing "only showing top 1 rows"
// +---+
// | a|
@ -116,17 +112,13 @@ public class LivySQLInterpreterTest {
// | 1|
// +---+
// only showing top 1 rows
rows = sqlInterpreter.parseSQLOutput("+---+\n" +
"| a|\n" +
"+---+\n" +
"| 1|\n" +
"+---+\n" +
"only showing top 1 rows");
rows =
sqlInterpreter.parseSQLOutput(
"+---+\n" + "| a|\n" + "+---+\n" + "| 1|\n" + "+---+\n" + "only showing top 1 rows");
assertEquals(2, rows.size());
assertEquals("a", rows.get(0));
assertEquals("1", rows.get(1));
// sql output with 3 rows, 3 columns, showing "only showing top 3 rows" with a line break in
// the data
// +---+---+---+
@ -138,21 +130,22 @@ public class LivySQLInterpreterTest {
// | 3a| 3b| 3c|
// +---+---+---+
// only showing top 3 rows
rows = sqlInterpreter.parseSQLOutput("+---+----+---+\n" +
"| a| b| c|\n" +
"+---+----+---+\n" +
"| 1a| 1b| 1c|\n" +
"| 2a| 2\nb| 2c|\n" +
"| 3a| 3b| 3c|\n" +
"+---+---+---+\n" +
"only showing top 3 rows");
rows =
sqlInterpreter.parseSQLOutput(
"+---+----+---+\n"
+ "| a| b| c|\n"
+ "+---+----+---+\n"
+ "| 1a| 1b| 1c|\n"
+ "| 2a| 2\nb| 2c|\n"
+ "| 3a| 3b| 3c|\n"
+ "+---+---+---+\n"
+ "only showing top 3 rows");
assertEquals(4, rows.size());
assertEquals("a\tb\tc", rows.get(0));
assertEquals("1a\t1b\t1c", rows.get(1));
assertEquals("2a\t2\\nb\t2c", rows.get(2));
assertEquals("3a\t3b\t3c", rows.get(3));
// sql output with 2 rows and one containing a tab
// +---+---+
// | a| b|
@ -160,12 +153,14 @@ public class LivySQLInterpreterTest {
// | 1| \ta|
// | 2| 2b|
// +---+---+
rows = sqlInterpreter.parseSQLOutput("+---+---+\n" +
"| a| b|\n" +
"+---+---+\n" +
"| 1| \ta|\n" +
"| 2| 2b|\n" +
"+---+---+");
rows =
sqlInterpreter.parseSQLOutput(
"+---+---+\n"
+ "| a| b|\n"
+ "+---+---+\n"
+ "| 1| \ta|\n"
+ "| 2| 2b|\n"
+ "+---+---+");
assertEquals(3, rows.size());
assertEquals("a\tb", rows.get(0));
assertEquals("1\t\\ta", rows.get(1));

7
markdown/pom.xml
View file

@ -93,13 +93,6 @@
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

24
markdown/src/main/java/org/apache/zeppelin/markdown/Markdown.java
View file

@ -17,12 +17,8 @@
package org.apache.zeppelin.markdown;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -31,18 +27,16 @@ import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* MarkdownInterpreter interpreter for Zeppelin.
*/
/** MarkdownInterpreter interpreter for Zeppelin. */
public class Markdown extends Interpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(Markdown.class);
private MarkdownParser parser;
/**
* Markdown Parser Type.
*/
/** Markdown Parser Type. */
public enum MarkdownParserType {
PEGDOWN {
@Override
@ -85,8 +79,7 @@ public class Markdown extends Interpreter {
}
@Override
public void close() {
}
public void close() {}
@Override
public InterpreterResult interpret(String markdownText, InterpreterContext interpreterContext) {
@ -103,8 +96,7 @@ public class Markdown extends Interpreter {
}
@Override
public void cancel(InterpreterContext context) {
}
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
@ -123,8 +115,8 @@ public class Markdown extends Interpreter {
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
public List<InterpreterCompletion> completion(
String buf, int cursor, InterpreterContext interpreterContext) {
return null;
}
}

Some files were not shown because too many files have changed in this diff Show more