mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Merge remote-tracking branch 'origin/master' into ZEPPELIN-2216
# Conflicts: # jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
This commit is contained in:
Tinkoff DWH
commit
39777221dc
288 changed files with 14508 additions and 7042 deletions
16
.travis.yml
16
.travis.yml
|
|
@ -37,7 +37,7 @@ addons:
|
|||
env:
|
||||
global:
|
||||
# Interpreters does not required by zeppelin-server integration tests
|
||||
- INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy'
|
||||
- INTERPRETERS='!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!python,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy'
|
||||
|
||||
matrix:
|
||||
include:
|
||||
|
|
@ -51,7 +51,7 @@ matrix:
|
|||
|
||||
# Test selenium with spark module for 1.6.3
|
||||
- jdk: "oraclejdk7"
|
||||
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
|
||||
# Test interpreter modules
|
||||
- jdk: "oraclejdk7"
|
||||
|
|
@ -59,27 +59,27 @@ matrix:
|
|||
|
||||
# Test spark module for 2.1.0 with scala 2.11, livy
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 2.0.2 with scala 2.11
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.6.3 with scala 2.10
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.10" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.6.3 with scala 2.11
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test python/pyspark with python 2
|
||||
- jdk: "oraclejdk7"
|
||||
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
|
||||
# Test python/pyspark with python 3
|
||||
- jdk: "oraclejdk7"
|
||||
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
|
||||
before_install:
|
||||
# check files included in commit range, clear bower_components if a bower.json file has changed.
|
||||
|
|
|
|||
1
LICENSE
1
LICENSE
|
|
@ -255,6 +255,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
|
|||
(Apache 2.0) Bootstrap v3.0.2 (http://getbootstrap.com/) - https://github.com/twbs/bootstrap/blob/v3.0.2/LICENSE
|
||||
(Apache 2.0) Software under ./bigquery/* was developed at Google (http://www.google.com/). Licensed under the Apache v2.0 License.
|
||||
(Apache 2.0) Roboto Font (https://github.com/google/roboto/)
|
||||
(Apache 2.0) Gson extra (https://github.com/DanySK/gson-extras)
|
||||
|
||||
========================================================================
|
||||
BSD 3-Clause licenses
|
||||
|
|
|
|||
|
|
@ -23,9 +23,9 @@ import java.io.PrintStream;
|
|||
import java.io.ByteArrayOutputStream;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
|
|
@ -166,7 +166,8 @@ public class AlluxioInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " ");
|
||||
String lastWord = "";
|
||||
if (words.length > 0) {
|
||||
|
|
@ -176,7 +177,7 @@ public class AlluxioInterpreter extends Interpreter {
|
|||
List<InterpreterCompletion> voices = new LinkedList<>();
|
||||
for (String command : keywords) {
|
||||
if (command.startsWith(lastWord)) {
|
||||
voices.add(new InterpreterCompletion(command, command));
|
||||
voices.add(new InterpreterCompletion(command, command, CompletionType.command.name()));
|
||||
}
|
||||
}
|
||||
return voices;
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ import java.util.Properties;
|
|||
|
||||
import alluxio.client.WriteType;
|
||||
import alluxio.client.file.URIStatus;
|
||||
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
|
|
@ -78,28 +80,28 @@ public class AlluxioInterpreterTest {
|
|||
@Test
|
||||
public void testCompletion() {
|
||||
List expectedResultOne = Arrays.asList(
|
||||
new InterpreterCompletion("cat", "cat"),
|
||||
new InterpreterCompletion("chgrp", "chgrp"),
|
||||
new InterpreterCompletion("chmod", "chmod"),
|
||||
new InterpreterCompletion("chown", "chown"),
|
||||
new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
|
||||
new InterpreterCompletion("copyToLocal", "copyToLocal"),
|
||||
new InterpreterCompletion("count", "count"),
|
||||
new InterpreterCompletion("createLineage", "createLineage"));
|
||||
new InterpreterCompletion("cat", "cat", CompletionType.command.name()),
|
||||
new InterpreterCompletion("chgrp", "chgrp", CompletionType.command.name()),
|
||||
new InterpreterCompletion("chmod", "chmod", CompletionType.command.name()),
|
||||
new InterpreterCompletion("chown", "chown", CompletionType.command.name()),
|
||||
new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
|
||||
new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
|
||||
new InterpreterCompletion("count", "count", CompletionType.command.name()),
|
||||
new InterpreterCompletion("createLineage", "createLineage", CompletionType.command.name()));
|
||||
List expectedResultTwo = Arrays.asList(
|
||||
new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
|
||||
new InterpreterCompletion("copyToLocal", "copyToLocal"),
|
||||
new InterpreterCompletion("count", "count"));
|
||||
new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
|
||||
new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()),
|
||||
new InterpreterCompletion("count", "count", CompletionType.command.name()));
|
||||
List expectedResultThree = Arrays.asList(
|
||||
new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
|
||||
new InterpreterCompletion("copyToLocal", "copyToLocal"));
|
||||
new InterpreterCompletion("copyFromLocal", "copyFromLocal", CompletionType.command.name()),
|
||||
new InterpreterCompletion("copyToLocal", "copyToLocal", CompletionType.command.name()));
|
||||
List expectedResultNone = new ArrayList<>();
|
||||
|
||||
List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0);
|
||||
List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0);
|
||||
List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0);
|
||||
List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0);
|
||||
List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0);
|
||||
List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0, null);
|
||||
List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0, null);
|
||||
List<InterpreterCompletion> resultThree = alluxioInterpreter.completion("copy", 0, null);
|
||||
List<InterpreterCompletion> resultNotMatch = alluxioInterpreter.completion("notMatch", 0, null);
|
||||
List<InterpreterCompletion> resultAll = alluxioInterpreter.completion("", 0, null);
|
||||
|
||||
Assert.assertEquals(expectedResultOne, resultOne);
|
||||
Assert.assertEquals(expectedResultTwo, resultTwo);
|
||||
|
|
|
|||
|
|
@ -67,7 +67,8 @@ public class AngularInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return new LinkedList<>();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -92,7 +92,8 @@ public class BeamInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -332,7 +332,8 @@ public class BigQueryInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return NO_COMPLETION;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ fi
|
|||
|
||||
. "${bin}/common.sh"
|
||||
|
||||
ZEPPELIN_INTP_CLASSPATH=""
|
||||
ZEPPELIN_INTP_CLASSPATH="${CLASSPATH}"
|
||||
|
||||
# construct classpath
|
||||
if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
|
||||
|
|
@ -190,8 +190,6 @@ fi
|
|||
|
||||
addJarInDirForIntp "${LOCAL_INTERPRETER_REPO}"
|
||||
|
||||
CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
|
||||
|
||||
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
|
||||
suid="$(id -u ${ZEPPELIN_IMPERSONATE_USER})"
|
||||
if [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then
|
||||
|
|
@ -204,12 +202,12 @@ fi
|
|||
|
||||
if [[ -n "${SPARK_SUBMIT}" ]]; then
|
||||
if [[ -n "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" != "false" ]]; then
|
||||
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}`
|
||||
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}`
|
||||
else
|
||||
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT}`
|
||||
INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT}`
|
||||
fi
|
||||
else
|
||||
INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} `
|
||||
INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} `
|
||||
fi
|
||||
|
||||
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ -n "${suid}" || -z "${SPARK_SUBMIT}" ]]; then
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib"
|
|||
addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
|
||||
addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib"
|
||||
|
||||
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
|
||||
ZEPPELIN_CLASSPATH="$CLASSPATH:$ZEPPELIN_CLASSPATH"
|
||||
|
||||
if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
|
||||
echo "Log dir doesn't exist, create ${ZEPPELIN_LOG_DIR}"
|
||||
|
|
@ -83,4 +83,4 @@ if [[ ! -d "${ZEPPELIN_PID_DIR}" ]]; then
|
|||
$(mkdir -p "${ZEPPELIN_PID_DIR}")
|
||||
fi
|
||||
|
||||
exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:$CLASSPATH $ZEPPELIN_SERVER "$@"
|
||||
exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:${ZEPPELIN_CLASSPATH} $ZEPPELIN_SERVER "$@"
|
||||
|
|
|
|||
|
|
@ -216,7 +216,8 @@ public class CassandraInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return NO_COMPLETION;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import com.datastax.driver.core.exceptions.DriverException
|
|||
import com.datastax.driver.core.policies.{LoggingRetryPolicy, FallthroughRetryPolicy, DowngradingConsistencyRetryPolicy, Policies}
|
||||
import org.apache.zeppelin.cassandra.TextBlockHierarchy._
|
||||
import org.apache.zeppelin.display.AngularObjectRegistry
|
||||
import org.apache.zeppelin.display.Input.ParamOption
|
||||
import org.apache.zeppelin.display.ui.OptionInput.ParamOption
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code
|
||||
import org.apache.zeppelin.interpreter.{InterpreterException, InterpreterResult, InterpreterContext}
|
||||
import org.slf4j.LoggerFactory
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ import com.datastax.driver.core.Statement;
|
|||
|
||||
import org.apache.zeppelin.display.AngularObjectRegistry;
|
||||
import org.apache.zeppelin.display.GUI;
|
||||
import org.apache.zeppelin.display.Input.ParamOption;
|
||||
import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterException;
|
||||
import org.junit.Rule;
|
||||
|
|
|
|||
|
|
@ -259,13 +259,13 @@
|
|||
|
||||
<property>
|
||||
<name>zeppelin.interpreters</name>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter</value>
|
||||
<description>Comma separated interpreter configurations. First interpreter become a default</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>zeppelin.interpreter.group.order</name>
|
||||
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam</value>
|
||||
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam,groovy</value>
|
||||
<description></description>
|
||||
</property>
|
||||
|
||||
|
|
|
|||
|
|
@ -106,8 +106,8 @@ function make_binary_release() {
|
|||
|
||||
git_clone
|
||||
make_source_package
|
||||
make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION}"
|
||||
make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
|
||||
make_binary_release all "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION}"
|
||||
make_binary_release netinst "-Pspark-2.1 -Phadoop-2.6 -Pscala-${SCALA_VERSION} -pl zeppelin-interpreter,zeppelin-zengine,:zeppelin-display_${SCALA_VERSION},:zeppelin-spark-dependencies_${SCALA_VERSION},:zeppelin-spark_${SCALA_VERSION},zeppelin-web,zeppelin-server,zeppelin-distribution -am"
|
||||
|
||||
# remove non release files and dirs
|
||||
rm -rf "${WORKING_DIR}/zeppelin"
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ if [[ $RELEASE_VERSION == *"SNAPSHOT"* ]]; then
|
|||
DO_SNAPSHOT="yes"
|
||||
fi
|
||||
|
||||
PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pyarn -Ppyspark -Psparkr -Pr"
|
||||
PUBLISH_PROFILES="-Ppublish-distr -Pspark-2.1 -Phadoop-2.6 -Pr"
|
||||
PROJECT_OPTIONS="-pl !zeppelin-distribution"
|
||||
NEXUS_STAGING="https://repository.apache.org/service/local/staging"
|
||||
NEXUS_PROFILE="153446d1ac37c4"
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@
|
|||
<li><a href="{{BASE_PATH}}/interpreter/elasticsearch.html">Elasticsearch</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/flink.html">Flink</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/geode.html">Geode</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/groovy.html">Groovy</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/hbase.html">HBase</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/hdfs.html">HDFS</a></li>
|
||||
<li><a href="{{BASE_PATH}}/interpreter/hive.html">Hive</a></li>
|
||||
|
|
@ -116,10 +117,11 @@
|
|||
<li><a href="{{BASE_PATH}}/security/notebook_authorization.html">Notebook Authorization</a></li>
|
||||
<li><a href="{{BASE_PATH}}/security/datasource_authorization.html">Data Source Authorization</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li class="title"><span><b>Helium Framework</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application (Experimental)</a></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelinspell.html">Writing Zeppelin Spell (Experimental)</a></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization (Experimental)</a></li>
|
||||
<li class="title"><span><b>Helium Framework (Experimental)</b></span></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelinapplication.html">Writing Zeppelin Application</a></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelinspell.html">Writing Zeppelin Spell</a></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization.html">Writing Zeppelin Visualization: Basics</a></li>
|
||||
<li><a href="{{BASE_PATH}}/development/writingzeppelinvisualization_transformation.html">Writing Zeppelin Visualization: Transformation</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li class="title"><span><b>Advanced</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/install/virtual_machine.html">Zeppelin on Vagrant VM</a></li>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Writing a new Application(Experimental)"
|
||||
title: "Writing a new Application"
|
||||
description: "Apache Zeppelin Application is a package that runs on Interpreter process and displays it's output inside of the notebook. Make your own Application in Apache Zeppelin is quite easy."
|
||||
group: development
|
||||
---
|
||||
|
|
@ -19,7 +19,7 @@ limitations under the License.
|
|||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Writing a new Application (Experimental)
|
||||
# Writing a new Application
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
|
|
@ -91,19 +91,18 @@ In the Zeppelin notebook, run `%dev run` will connect to application running in
|
|||
Package file is a json file that provides information about the application.
|
||||
Json file contains the following information
|
||||
|
||||
```
|
||||
```json
|
||||
{
|
||||
name : "[organization].[name]",
|
||||
description : "Description",
|
||||
artifact : "groupId:artifactId:version",
|
||||
className : "your.package.name.YourApplicationClass",
|
||||
resources : [
|
||||
"name" : "[organization].[name]",
|
||||
"description" : "Description",
|
||||
"artifact" : "groupId:artifactId:version",
|
||||
"className" : "your.package.name.YourApplicationClass",
|
||||
"resources" : [
|
||||
["resource.name", ":resource.class.name"],
|
||||
["alternative.resource.name", ":alternative.class.name"]
|
||||
],
|
||||
icon : "<i class="icon"></i>"
|
||||
"icon" : "<i class='icon'></i>"
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
#### name
|
||||
|
|
@ -184,4 +183,3 @@ e.g.
|
|||
```
|
||||
icon: "<i class='fa fa-clock-o'></i>"
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Writing a new Spell(Experimental)"
|
||||
title: "Writing a new Spell"
|
||||
description: "Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter. "
|
||||
group: development
|
||||
---
|
||||
|
|
@ -19,16 +19,16 @@ limitations under the License.
|
|||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Writing a new Spell (Experimental)
|
||||
# Writing a new Spell
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
## What is Apache Zeppelin Spell
|
||||
## What is Apache Zeppelin Spell
|
||||
|
||||
Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter.
|
||||
Spell is a kind of interpreter that runs on browser not on backend. So, technically it's the frontend interpreter.
|
||||
It can provide many benefits.
|
||||
|
||||
- Spell is pluggable frontend interpreter. So it can be installed and removed easily using helium registry.
|
||||
- Spell is pluggable frontend interpreter. So it can be installed and removed easily using helium registry.
|
||||
- Every spell is written in javascript. It means you can use existing javascript libraries whatever you want.
|
||||
- Spell runs on browser like display system (`%html`, `%table`). In other words, every spell can be used as display system as well.
|
||||
|
||||
|
|
@ -57,7 +57,7 @@ For example, Use `%echo` for the Echo Spell.
|
|||
<img class="img-responsive" style="width:70%" src="../assets/themes/zeppelin/img/docs-img/writing_spell_using.gif" />
|
||||
|
||||
|
||||
## Write a new Spell
|
||||
## Write a new Spell
|
||||
|
||||
Making a new spell is similar to [Helium Visualization#write-new-visualization](./writingzeppelinvisualization.html#write-new-visualization).
|
||||
|
||||
|
|
@ -67,14 +67,14 @@ Making a new spell is similar to [Helium Visualization#write-new-visualization](
|
|||
|
||||
### 1. Create a npm package
|
||||
|
||||
Create a [package.json](https://docs.npmjs.com/files/package.json) in new directory for spell.
|
||||
Create a [package.json](https://docs.npmjs.com/files/package.json) in new directory for spell.
|
||||
|
||||
- You have to add a framework called `zeppelin-spell` as a dependency to create spell ([zeppelin-spell](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/spell))
|
||||
- Also, you can add any dependencies you want to utilise.
|
||||
|
||||
Here's an example
|
||||
|
||||
```
|
||||
```json
|
||||
{
|
||||
"name": "zeppelin-echo-spell",
|
||||
"description": "Zeppelin Echo Spell (example)",
|
||||
|
|
@ -95,7 +95,7 @@ Here's an example
|
|||
}
|
||||
```
|
||||
|
||||
### 2. Write spell using framework
|
||||
### 2. Write spell using framework
|
||||
|
||||
Here are some examples you can refer
|
||||
|
||||
|
|
@ -106,7 +106,7 @@ Here are some examples you can refer
|
|||
|
||||
Now, you need to write code to create spell which processing text.
|
||||
|
||||
```javascript
|
||||
```js
|
||||
import {
|
||||
SpellBase,
|
||||
SpellResult,
|
||||
|
|
@ -121,8 +121,8 @@ export default class EchoSpell extends SpellBase {
|
|||
|
||||
interpret(paragraphText) {
|
||||
const processed = paragraphText + '!';
|
||||
|
||||
/**
|
||||
|
||||
/**
|
||||
* should return `SpellResult` which including `data` and `type`
|
||||
* default type is `TEXT` if you don't specify.
|
||||
*/
|
||||
|
|
@ -133,7 +133,7 @@ export default class EchoSpell extends SpellBase {
|
|||
|
||||
Here is another example. Let's say we want to create markdown spell. First of all, we should add a dependency for markdown in package.json
|
||||
|
||||
```
|
||||
```json
|
||||
// package.json
|
||||
"dependencies": {
|
||||
"markdown": "0.5.0",
|
||||
|
|
@ -143,7 +143,7 @@ Here is another example. Let's say we want to create markdown spell. First of al
|
|||
|
||||
And here is spell code.
|
||||
|
||||
```javascript
|
||||
```js
|
||||
import {
|
||||
SpellBase,
|
||||
SpellResult,
|
||||
|
|
@ -171,16 +171,16 @@ export default class MarkdownSpell extends SpellBase {
|
|||
}
|
||||
```
|
||||
|
||||
- You might want to manipulate DOM directly (e.g google d3.js), then refer [Flowchart Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-flowchart/index.js)
|
||||
- You might want to manipulate DOM directly (e.g google d3.js), then refer [Flowchart Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-flowchart/index.js)
|
||||
- You might want to return promise not string (e.g API call), then refer [Google Translation API Spell](https://github.com/apache/zeppelin/blob/master/zeppelin-examples/zeppelin-example-spell-translator/index.js)
|
||||
|
||||
### 3. Create __Helium package file__ for local deployment
|
||||
### 3. Create __Helium package file__ for local deployment
|
||||
|
||||
You don't want to publish your package every time you make a change in your spell. Zeppelin provides local deploy.
|
||||
The only thing you need to do is creating a __Helium Package file__ (JSON) for local deploy.
|
||||
The only thing you need to do is creating a __Helium Package file__ (JSON) for local deploy.
|
||||
It's automatically created when you publish to npm repository but in local case, you should make it by yourself.
|
||||
|
||||
```
|
||||
```json
|
||||
{
|
||||
"type" : "SPELL",
|
||||
"name" : "zeppelin-echo-spell",
|
||||
|
|
@ -197,18 +197,18 @@ It's automatically created when you publish to npm repository but in local case,
|
|||
|
||||
- Place this file in your local registry directory (default `$ZEPPELIN_HOME/helium`).
|
||||
- `type` should be `SPELL`
|
||||
- Make sure that `artifact` should be same as your spell directory.
|
||||
- Make sure that `artifact` should be same as your spell directory.
|
||||
- You can get information about other fields in [Helium Visualization#3-create-helium-package-file-and-locally-deploy](./writingzeppelinvisualization.html#3-create-helium-package-file-and-locally-deploy).
|
||||
|
||||
### 4. Run in dev mode
|
||||
|
||||
```
|
||||
```bash
|
||||
cd zeppelin-web
|
||||
yarn run dev:helium
|
||||
yarn run dev:helium
|
||||
```
|
||||
|
||||
You can browse localhost:9000. Every time refresh your browser, Zeppelin will rebuild your spell and reload changes.
|
||||
|
||||
### 5. Publish your spell to the npm repository
|
||||
### 5. Publish your spell to the npm repository
|
||||
|
||||
See [Publishing npm packages](https://docs.npmjs.com/getting-started/publishing-npm-packages)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Writing a new Visualization(Experimental)"
|
||||
title: "Writing a new Visualization"
|
||||
description: "Apache Zeppelin Visualization is a pluggable package that can be loaded/unloaded on runtime through Helium framework in Zeppelin. A Visualization is a javascript npm package and user can use them just like any other built-in visualization in a note."
|
||||
group: development
|
||||
---
|
||||
|
|
@ -19,7 +19,7 @@ limitations under the License.
|
|||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Writing a new Visualization (Experimental)
|
||||
# Writing a new Visualization
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
|
|
@ -61,11 +61,11 @@ User can use just like any other built-in visualizations.
|
|||
|
||||
#### 1. Create a npm package
|
||||
|
||||
Create a [package.json](https://docs.npmjs.com/files/package.json) in your new Visualization directory. Normally, you can add any dependencies in package.json however Zeppelin Visualization package only allows two dependencies: [zeppelin-vis](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/visualization) and [zeppelin-tabledata](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/tabledata).
|
||||
Create a [package.json](https://docs.npmjs.com/files/package.json) in your new Visualization directory. You can add any dependencies in package.json, but you **must include two dependencies: [zeppelin-vis](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/visualization) and [zeppelin-tabledata](https://github.com/apache/zeppelin/tree/master/zeppelin-web/src/app/tabledata).**
|
||||
|
||||
Here's an example
|
||||
|
||||
```
|
||||
```json
|
||||
{
|
||||
"name": "zeppelin_horizontalbar",
|
||||
"description" : "Horizontal Bar chart",
|
||||
|
|
@ -86,7 +86,7 @@ To create your own visualization, you need to create a js file and import [Visua
|
|||
|
||||
[Visualization](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/visualization/visualization.js) class, there're several methods that you need to override and implement. Here's simple visualization that just prints `Hello world`.
|
||||
|
||||
```
|
||||
```js
|
||||
import Visualization from 'zeppelin-vis'
|
||||
import PassthroughTransformation from 'zeppelin-tabledata/passthrough'
|
||||
|
||||
|
|
@ -118,7 +118,7 @@ Zeppelin's built-in visualization uses the same API, so you can check [built-in
|
|||
__Helium Package file__ is a json file that provides information about the application.
|
||||
Json file contains the following information
|
||||
|
||||
```
|
||||
```json
|
||||
{
|
||||
"type" : "VISUALIZATION",
|
||||
"name" : "zeppelin_horizontalbar",
|
||||
|
|
@ -154,15 +154,15 @@ e.g.
|
|||
|
||||
When artifact exists in npm repository
|
||||
|
||||
```
|
||||
artifact: "my-visualiztion@1.0.0"
|
||||
```json
|
||||
"artifact": "my-visualiztion@1.0.0"
|
||||
```
|
||||
|
||||
|
||||
When artifact exists in local file system
|
||||
|
||||
```
|
||||
artifact: "/path/to/my/visualization"
|
||||
```json
|
||||
"artifact": "/path/to/my/visualization"
|
||||
```
|
||||
|
||||
##### license
|
||||
|
|
@ -171,8 +171,8 @@ License information.
|
|||
|
||||
e.g.
|
||||
|
||||
```
|
||||
license: "Apache-2.0"
|
||||
```json
|
||||
"license": "Apache-2.0"
|
||||
```
|
||||
|
||||
##### icon
|
||||
|
|
@ -181,8 +181,8 @@ Icon to be used in visualization select button. String in this field will be ren
|
|||
|
||||
e.g.
|
||||
|
||||
```
|
||||
icon: "<i class='fa fa-coffee'></i>"
|
||||
```json
|
||||
"icon": "<i class='fa fa-coffee'></i>"
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -191,9 +191,9 @@ icon: "<i class='fa fa-coffee'></i>"
|
|||
Place your __Helium package file__ in local registry (ZEPPELIN_HOME/helium).
|
||||
Run Zeppelin. And then run zeppelin-web in visualization dev mode.
|
||||
|
||||
```
|
||||
```bash
|
||||
cd zeppelin-web
|
||||
yarn run dev:helium
|
||||
yarn run dev:helium
|
||||
```
|
||||
|
||||
You can browse localhost:9000. Everytime refresh your browser, Zeppelin will rebuild your visualization and reload changes.
|
||||
|
|
@ -202,4 +202,4 @@ You can browse localhost:9000. Everytime refresh your browser, Zeppelin will reb
|
|||
#### 5. Publish your visualization
|
||||
|
||||
Once it's done, publish your visualization package using `npm publish`.
|
||||
That's it. With in an hour, your visualization will be available in Zeppelin's helium menu.
|
||||
That's it. With in an hour, your visualization will be available in Zeppelin's helium menu.
|
||||
|
|
|
|||
281
docs/development/writingzeppelinvisualization_transformation.md
Normal file
281
docs/development/writingzeppelinvisualization_transformation.md
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Transformations for Zeppelin Visualization"
|
||||
description: "Description for Transformations"
|
||||
group: development
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Transformations for Zeppelin Visualization
|
||||
|
||||
<div id="toc"></div>
|
||||
|
||||
## Overview
|
||||
|
||||
Transformations
|
||||
|
||||
- **renders** setting which allows users to set columns and
|
||||
- **transforms** table rows according to the configured columns.
|
||||
|
||||
Zeppelin provides 4 types of transformations.
|
||||
|
||||
## 1. PassthroughTransformation
|
||||
|
||||
`PassthroughTransformation` is the simple transformation which does not convert original tabledata at all.
|
||||
|
||||
See [passthrough.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/passthrough.js)
|
||||
|
||||
## 2. ColumnselectorTransformation
|
||||
|
||||
`ColumnselectorTransformation` is uses when you need `N` axes but do not need aggregation.
|
||||
|
||||
See [columnselector.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/columnselector.js)
|
||||
|
||||
## 3. PivotTransformation
|
||||
|
||||
`PivotTransformation` provides group by and aggregation. Every chart using `PivotTransformation` has 3 axes. `Keys`, `Groups` and `Values`.
|
||||
|
||||
See [pivot.js](https://github.com/apache/zeppelin/blob/master/zeppelin-web/src/app/tabledata/pivot.js)
|
||||
|
||||
## 4. AdvancedTransformation
|
||||
|
||||
`AdvancedTransformation` has more detailed options while providing existing features of `PivotTransformation` and `ColumnselectorTransformation`
|
||||
|
||||
- multiple sub charts
|
||||
- configurable chart axes
|
||||
- parameter widgets: `input`, `checkbox`, `option`, `textarea`
|
||||
- parsing parameters automatically based on their types
|
||||
- expand / fold axis and parameter panels
|
||||
- multiple transformation methods while supporting lazy converting
|
||||
- re-initialize the whole configuration based on spec hash.
|
||||
|
||||
### Spec
|
||||
|
||||
`AdvancedTransformation` requires `spec` which includes axis and parameter details for charts.
|
||||
|
||||
Let's create 2 sub-charts called `line` and `no-group`. Each sub chart can have different axis and parameter depending on their requirements.
|
||||
|
||||
<br/>
|
||||
|
||||
```js
|
||||
class AwesomeVisualization extends Visualization {
|
||||
constructor(targetEl, config) {
|
||||
super(targetEl, config)
|
||||
|
||||
const spec = {
|
||||
charts: {
|
||||
'line': {
|
||||
transform: { method: 'object', },
|
||||
sharedAxis: false, /** set if you want to share axes between sub charts, default is `false` */
|
||||
axis: {
|
||||
'xAxis': { dimension: 'multiple', axisType: 'key', description: 'serial', },
|
||||
'yAxis': { dimension: 'multiple', axisType: 'aggregator', description: 'serial', },
|
||||
'category': { dimension: 'multiple', axisType: 'group', description: 'categorical', },
|
||||
},
|
||||
parameter: {
|
||||
'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
|
||||
'yAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of yAxis', },
|
||||
'lineWidth': { valueType: 'int', defaultValue: 0, description: 'width of line', },
|
||||
},
|
||||
},
|
||||
|
||||
'no-group': {
|
||||
transform: { method: 'object', },
|
||||
sharedAxis: false,
|
||||
axis: {
|
||||
'xAxis': { dimension: 'single', axisType: 'key', },
|
||||
'yAxis': { dimension: 'multiple', axisType: 'value', },
|
||||
},
|
||||
parameter: {
|
||||
'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
|
||||
'yAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of yAxis', },
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
this.transformation = new AdvancedTransformation(config, spec)
|
||||
}
|
||||
|
||||
...
|
||||
|
||||
// `render` will be called whenever `axis` or `parameter` is changed
|
||||
render(data) {
|
||||
const { chart, parameter, column, transformer, } = data
|
||||
|
||||
if (chart === 'line') {
|
||||
const transformed = transformer()
|
||||
// draw line chart
|
||||
} else if (chart === 'no-group') {
|
||||
const transformed = transformer()
|
||||
// draw no-group chart
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
<br/>
|
||||
|
||||
### Spec: `axis`
|
||||
|
||||
| Field Name | Available Values (type) | Description |
|
||||
| --- | --- | --- |
|
||||
|`dimension` | `single` | Axis can contains only 1 column |
|
||||
|`dimension` | `multiple` | Axis can contains multiple columns |
|
||||
|`axisType` | `key` | Column(s) in this axis will be used as `key` like in `PivotTransformation`. These columns will be served in `column.key` |
|
||||
|`axisType` | `aggregator` | Column(s) in this axis will be used as `value` like in `PivotTransformation`. These columns will be served in `column.aggregator` |
|
||||
|`axisType` | `group` | Column(s) in this axis will be used as `group` like in `PivotTransformation`. These columns will be served in `column.group` |
|
||||
|`axisType` | (string) | Any string value can be used here. These columns will be served in `column.custom` |
|
||||
|`maxAxisCount` (optional) | (int) | The max number of columns that this axis can contain. (unlimited if `undefined`) |
|
||||
|`minAxisCount` (optional) | (int) | The min number of columns that this axis should contain to draw chart. (`1` in case of single dimension) |
|
||||
|`description` (optional) | (string) | Description for the axis. |
|
||||
|
||||
<br/>
|
||||
|
||||
Here is an example.
|
||||
|
||||
```js
|
||||
axis: {
|
||||
'xAxis': { dimension: 'multiple', axisType: 'key', },
|
||||
'yAxis': { dimension: 'multiple', axisType: 'aggregator'},
|
||||
'category': { dimension: 'multiple', axisType: 'group', maxAxisCount: 2, valueType: 'string', },
|
||||
},
|
||||
```
|
||||
|
||||
<br/>
|
||||
|
||||
### Spec: `sharedAxis`
|
||||
|
||||
If you set `sharedAxis: false` for sub charts, then their axes are persisted in global space (shared). It's useful for when you creating multiple sub charts sharing their axes but have different parameters. For example,
|
||||
|
||||
- `basic-column`, `stacked-column`, `percent-column`
|
||||
- `pie` and `donut`
|
||||
|
||||
<br/>
|
||||
|
||||
Here is an example.
|
||||
|
||||
```js
|
||||
const spec = {
|
||||
charts: {
|
||||
'column': {
|
||||
transform: { method: 'array', },
|
||||
sharedAxis: true,
|
||||
axis: { ... },
|
||||
parameter: { ... },
|
||||
},
|
||||
|
||||
'stacked': {
|
||||
transform: { method: 'array', },
|
||||
sharedAxis: true,
|
||||
axis: { ... }
|
||||
parameter: { ... },
|
||||
},
|
||||
```
|
||||
|
||||
<br/>
|
||||
|
||||
### Spec: `parameter`
|
||||
|
||||
| Field Name | Available Values (type) | Description |
|
||||
| --- | --- | --- |
|
||||
|`valueType` | `string` | Parameter which has string value |
|
||||
|`valueType` | `int` | Parameter which has int value |
|
||||
|`valueType` | `float` | Parameter which has float value |
|
||||
|`valueType` | `boolean` | Parameter which has boolean value used with `checkbox` widget usually |
|
||||
|`valueType` | `JSON` | Parameter which has JSON value used with `textarea` widget usually. `defaultValue` should be `""` (empty string). This ||`defaultValue` | (any) | Default value of this parameter. `JSON` type should have `""` (empty string) |
|
||||
|`description` | (string) | Description of this parameter. This value will be parsed as HTML for pretty output |
|
||||
|`widget` | `input` | Use [input](https://developer.mozilla.org/en/docs/Web/HTML/Element/input) widget. This is the default widget (if `widget` is undefined)|
|
||||
|`widget` | `checkbox` | Use [checkbox](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input/checkbox) widget. |
|
||||
|`widget` | `textarea` | Use [textarea](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/textarea) widget. |
|
||||
|`widget` | `option` | Use [select + option](https://developer.mozilla.org/en-US/docs/Web/HTML/Element/select) widget. This parameter should have `optionValues` field as well. |
|
||||
|`optionValues` | (Array<string>) | Available option values used with the `option` widget |
|
||||
|
||||
<br/>
|
||||
|
||||
Here is an example.
|
||||
|
||||
```js
|
||||
parameter: {
|
||||
// string type, input widget
|
||||
'xAxisUnit': { valueType: 'string', defaultValue: '', description: 'unit of xAxis', },
|
||||
|
||||
// boolean type, checkbox widget
|
||||
'inverted': { widget: 'checkbox', valueType: 'boolean', defaultValue: false, description: 'invert x and y axes', },
|
||||
|
||||
// string type, option widget with `optionValues`
|
||||
'graphType': { widget: 'option', valueType: 'string', defaultValue: 'line', description: 'graph type', optionValues: [ 'line', 'smoothedLine', 'step', ], },
|
||||
|
||||
// HTML in `description`
|
||||
'dateFormat': { valueType: 'string', defaultValue: '', description: 'format of date (<a href="https://docs.amcharts.com/3/javascriptcharts/AmGraph#dateFormat">doc</a>) (e.g YYYY-MM-DD)', },
|
||||
|
||||
// JSON type, textarea widget
|
||||
'yAxisGuides': { widget: 'textarea', valueType: 'JSON', defaultValue: '', description: 'guides of yAxis ', },
|
||||
```
|
||||
|
||||
<br/>
|
||||
|
||||
### Spec: `transform`
|
||||
|
||||
| Field Name | Available Values (type) | Description |
|
||||
| --- | --- | --- |
|
||||
|`method` | `object` | designed for rows requiring object manipulation |
|
||||
|`method` | `array` | designed for rows requiring array manipulation |
|
||||
|`method` | `array:2-key` | designed for xyz charts (e.g bubble chart) |
|
||||
|`method` | `drill-down` | designed for drill-down charts |
|
||||
|`method` | `raw` | will return the original `tableData.rows` |
|
||||
|
||||
<br/>
|
||||
|
||||
Whatever you specified as `transform.method`, the `transformer` value will be always function for lazy computation.
|
||||
|
||||
```js
|
||||
// advanced-transformation.util#getTransformer
|
||||
|
||||
if (transformSpec.method === 'raw') {
|
||||
transformer = () => { return rows; }
|
||||
} else if (transformSpec.method === 'array') {
|
||||
transformer = () => {
|
||||
...
|
||||
return { ... }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Here is actual usage.
|
||||
|
||||
```js
|
||||
class AwesomeVisualization extends Visualization {
|
||||
constructor(...) { /** setup your spec */ }
|
||||
|
||||
...
|
||||
|
||||
// `render` will be called whenever `axis` or `parameter` are changed
|
||||
render(data) {
|
||||
const { chart, parameter, column, transformer, } = data
|
||||
|
||||
if (chart === 'line') {
|
||||
const transformed = transformer()
|
||||
// draw line chart
|
||||
} else if (chart === 'no-group') {
|
||||
const transformed = transformer()
|
||||
// draw no-group chart
|
||||
}
|
||||
}
|
||||
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -172,10 +172,11 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor
|
|||
* [Shiro Authentication](./security/shiroauthentication.html)
|
||||
* [Notebook Authorization](./security/notebook_authorization.html)
|
||||
* [Data Source Authorization](./security/datasource_authorization.html)
|
||||
* Helium Framework
|
||||
* [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html)
|
||||
* [Writing Zeppelin Spell (Experimental)](./development/writingzeppelinspell.html)
|
||||
* [Writing Zeppelin Visualization (Experimental)](./development/writingzeppelinvisualization.html)
|
||||
* Helium Framework (Experimental)
|
||||
* [Writing Zeppelin Application](./development/writingzeppelinapplication.html)
|
||||
* [Writing Zeppelin Spell](./development/writingzeppelinspell.html)
|
||||
* [Writing Zeppelin Visualization: Basic](./development/writingzeppelinvisualization.html)
|
||||
* [Writing Zeppelin Visualization: Transformation](./development/writingzeppelinvisualization_transformation.html)
|
||||
* Advanced
|
||||
* [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html)
|
||||
* [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode)
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ If you're unsure about the options, use the same commands that creates official
|
|||
# update all pom.xml to use scala 2.11
|
||||
./dev/change_scala_version.sh 2.11
|
||||
# build zeppelin with all interpreters and include latest version of Apache spark support for local mode.
|
||||
mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr -Pscala-2.11
|
||||
mvn clean package -DskipTests -Pspark-2.0 -Phadoop-2.4 -Pr -Pscala-2.11
|
||||
```
|
||||
|
||||
####3. Done
|
||||
|
|
@ -140,23 +140,10 @@ Available profiles are
|
|||
-Pscala-2.11
|
||||
```
|
||||
|
||||
##### `-Pyarn` (optional)
|
||||
|
||||
enable YARN support for local mode
|
||||
> YARN for local mode is not supported for Spark v1.5.0 or higher. Set `SPARK_HOME` instead.
|
||||
|
||||
##### `-Ppyspark` (optional)
|
||||
|
||||
enable [PySpark](http://spark.apache.org/docs/latest/api/python/) support for local mode.
|
||||
|
||||
##### `-Pr` (optional)
|
||||
|
||||
enable [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration.
|
||||
|
||||
##### `-Psparkr` (optional)
|
||||
|
||||
another [R](https://www.r-project.org/) support with [SparkR](https://spark.apache.org/docs/latest/sparkr.html) integration as well as local mode support.
|
||||
|
||||
##### `-Pvendor-repo` (optional)
|
||||
|
||||
enable 3rd party vendor repository (cloudera)
|
||||
|
|
@ -188,14 +175,14 @@ Here are some examples with several options:
|
|||
```bash
|
||||
# build with spark-2.1, scala-2.11
|
||||
./dev/change_scala_version.sh 2.11
|
||||
mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11 -DskipTests
|
||||
mvn clean package -Pspark-2.1 -Phadoop-2.4 -Pscala-2.11 -DskipTests
|
||||
|
||||
# build with spark-2.0, scala-2.11
|
||||
./dev/change_scala_version.sh 2.11
|
||||
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pscala-2.11 -DskipTests
|
||||
mvn clean package -Pspark-2.0 -Phadoop-2.4 -Pscala-2.11 -DskipTests
|
||||
|
||||
# build with spark-1.6, scala-2.10
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -DskipTests
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests
|
||||
|
||||
# spark-cassandra integration
|
||||
mvn clean package -Pcassandra-spark-1.5 -Dhadoop.version=2.6.0 -Phadoop-2.6 -DskipTests -DskipTests
|
||||
|
|
@ -328,10 +315,10 @@ mvn clean package -Pbuild-distr
|
|||
To build a distribution with specific profiles, run:
|
||||
|
||||
```sh
|
||||
mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark
|
||||
mvn clean package -Pbuild-distr -Pspark-1.5 -Phadoop-2.4
|
||||
```
|
||||
|
||||
The profiles `-Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark` can be adjusted if you wish to build to a specific spark versions, or omit support such as `yarn`.
|
||||
The profiles `-Pspark-1.5 -Phadoop-2.4` can be adjusted if you wish to build to a specific spark versions.
|
||||
|
||||
The archive is generated under _`zeppelin-distribution/target`_ directory
|
||||
|
||||
|
|
|
|||
|
|
@ -60,4 +60,4 @@ So, copying `notebook` and `conf` directory should be enough.
|
|||
### Upgrading from Zeppelin 0.7 to 0.8
|
||||
|
||||
- From 0.8, we recommend to use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` instead of `zeppelin.pyspark.python` as `zeppelin.pyspark.python` only effects driver. You can use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` as using them in spark.
|
||||
|
||||
- From 0.8, depending on your device, the keyboard shortcut `Ctrl-L` or `Command-L` which goes to the line somewhere user wants is not supported.
|
||||
|
|
@ -110,7 +110,7 @@ This assumes you've already cloned the project either on the host machine in the
|
|||
|
||||
```
|
||||
cd /zeppelin
|
||||
mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests
|
||||
./bin/zeppelin-daemon.sh start
|
||||
```
|
||||
|
||||
|
|
|
|||
116
docs/interpreter/groovy.md
Normal file
116
docs/interpreter/groovy.md
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Apache Groovy Interpreter for Apache Zeppelin"
|
||||
description: "Apache Groovy is a powerful, optionally typed and dynamic language, with static-typing and static compilation capabilities, for the Java platform aimed at improving developer productivity thanks to a concise, familiar and easy to learn syntax."
|
||||
group: interpreter
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Groovy Interpreter for Apache Zeppelin
|
||||
|
||||
|
||||
### Samples
|
||||
|
||||
```groovy
|
||||
%groovy
|
||||
//get a parameter defined as z.angularBind('ngSearchParam', value, 'paragraph_id')
|
||||
//g is a context object for groovy to avoid mix with z object
|
||||
def param = g.angular('ngSearchParam')
|
||||
//send request https://www.googleapis.com/customsearch/v1?q=ngSearchParam_value
|
||||
def r = HTTP.get(
|
||||
//assume you defined the groovy interpreter property
|
||||
// `search_baseurl`='https://www.googleapis.com/customsearch/v1'
|
||||
//in groovy object o.getProperty('A') == o.'A' == o.A == o['A']
|
||||
url : g.search_baseurl,
|
||||
query: [ q: param ],
|
||||
headers: [
|
||||
'Accept':'application/json',
|
||||
//'Authorization:' : g.getProperty('search_auth'),
|
||||
]
|
||||
)
|
||||
//check response code
|
||||
if( r.response.code==200 ) {
|
||||
g.html().with{
|
||||
//g.html() renders %angular to output and returns groovy.xml.MarkupBuilder
|
||||
h2("the response ${r.response.code}")
|
||||
span( r.response.body )
|
||||
h2("headers")
|
||||
pre( r.response.headers.join('\n') )
|
||||
}
|
||||
} else {
|
||||
//just to show that it's possible to use println with multiline groovy string to render output
|
||||
println("""%angular
|
||||
<script> alert ("code=${r.response.code} \n msg=${r.response.message}") </script>
|
||||
""")
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
```groovy
|
||||
%groovy
|
||||
|
||||
//renders a table with headers a, b, c and two rows
|
||||
g.table(
|
||||
[
|
||||
['a','b','c'],
|
||||
['a1','b1','c1'],
|
||||
['a2','b2','c2'],
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
### the `g` object
|
||||
|
||||
* `g.angular(String name)`
|
||||
|
||||
Returns angular object by name. Look up notebook scope first and then global scope.
|
||||
|
||||
|
||||
* `g.angularBind(String name, Object value)`
|
||||
|
||||
Assign a new `value` into angular object `name`
|
||||
|
||||
|
||||
* `java.util.Properties g.getProperties()`
|
||||
|
||||
returns all properties defined for this interpreter
|
||||
|
||||
|
||||
* `String g.getProperty('PROPERTY_NAME')`
|
||||
```groovy
|
||||
g.PROPERTY_NAME
|
||||
g.'PROPERTY_NAME'
|
||||
g['PROPERTY_NAME']
|
||||
g.getProperties().getProperty('PROPERTY_NAME')
|
||||
```
|
||||
|
||||
All above the accessor to named property defined in groovy interpreter.
|
||||
In this case with name `PROPERTY_NAME`
|
||||
|
||||
|
||||
* `groovy.xml.MarkupBuilder g.html()`
|
||||
|
||||
Starts or continues rendering of `%angular` to output and returns [groovy.xml.MarkupBuilder](http://groovy-lang.org/processing-xml.html#_markupbuilder)
|
||||
MarkupBuilder is usefull to generate html (xml)
|
||||
|
||||
* `void g.table(obj)`
|
||||
|
||||
starts or continues rendering table rows.
|
||||
|
||||
obj: List(rows) of List(columns) where first line is a header
|
||||
|
||||
|
||||
|
||||
|
|
@ -123,6 +123,11 @@ The JDBC interpreter properties are defined by default like below.
|
|||
<td></td>
|
||||
<td>Some SQL which executes every time after initialization of the interpreter (see [Binding mode](../manual/interpreters.md#interpreter-binding-mode))</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>default.completer.schemaFilters</td>
|
||||
<td></td>
|
||||
<td>Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.
|
||||
|
|
@ -164,6 +169,10 @@ There are more JDBC interpreter properties you can specify like below.
|
|||
<td>zeppelin.jdbc.keytab.location</td>
|
||||
<td>The path to the keytab file</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>zeppelin.jdbc.auth.kerberos.proxy.enable</td>
|
||||
<td>When auth type is Kerberos, enable/disable Kerberos proxy with the login user to get the connection. Default value is true.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>default.jceks.file</td>
|
||||
<td>jceks store path (e.g: jceks://file/tmp/zeppelin.jceks)</td>
|
||||
|
|
|
|||
|
|
@ -140,6 +140,11 @@ You can also set other Spark properties which are not listed in the table. For a
|
|||
<td>true</td>
|
||||
<td>Import implicits, UDF collection, and sql if set true.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>zeppelin.spark.enableSupportedVersionCheck</td>
|
||||
<td>true</td>
|
||||
<td>Do not change - developer only setting, not for production use</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.
|
||||
|
|
|
|||
|
|
@ -130,12 +130,11 @@ mvn clean package -DskipTests -Pspark-1.6 -Dflink.version=1.1.3 -Pscala-2.10
|
|||
-`-Pscala-2.10` tells maven to build with Scala v2.10.
|
||||
|
||||
|
||||
**Note:** You may wish to include additional build flags such as `-Ppyspark` or `-Psparkr`. See [the build section of github for more details](https://github.com/apache/zeppelin#build).
|
||||
|
||||
**Note:** You can build against any version of Spark that has a Zeppelin build profile available. The key is to make sure you check out the matching version of Spark to build. At the time of this writing, Spark 1.6 was the most recent Spark version available.
|
||||
|
||||
**Note:** On build failures. Having installed Zeppelin close to 30 times now, I will tell you that sometimes the build fails for seemingly no reason.
|
||||
As long as you didn't edit any code, it is unlikely the build is failing because of something you did. What does tend to happen, is some dependency that maven is trying to download is unreachable. If your build fails on this step here are some tips:
|
||||
|
||||
- Don't get discouraged.
|
||||
- Scroll up and read through the logs. There will be clues there.
|
||||
- Retry (that is, run the `mvn clean package -DskipTests -Pspark-1.6` again)
|
||||
|
|
@ -154,7 +153,7 @@ Use `ifconfig` to determine the host machine's IP address. If you are not famili
|
|||
|
||||
Open a web-browser on a machine connected to the same network as the host (or in the host operating system if using a virtual machine). Navigate to http://`yourip`:8080, where yourip is the IP address you found in `ifconfig`.
|
||||
|
||||
See the [Zeppelin tutorial](../tutorial/tutorial.md) for basic Zeppelin usage. It is also advised that you take a moment to check out the tutorial notebook that is included with each Zeppelin install, and to familiarize yourself with basic notebook functionality.
|
||||
See the [Zeppelin tutorial](../tutorial/tutorial.html) for basic Zeppelin usage. It is also advised that you take a moment to check out the tutorial notebook that is included with each Zeppelin install, and to familiarize yourself with basic notebook functionality.
|
||||
|
||||
##### Flink Test
|
||||
Create a new notebook named "Flink Test" and copy and paste the following code.
|
||||
|
|
@ -417,6 +416,6 @@ You should be able check the Flink and Spark webuis (at something like http://`y
|
|||
|
||||
### Next Steps
|
||||
|
||||
Check out the [tutorial](./tutorial.md) for more cool things you can do with your new toy!
|
||||
Check out the [tutorial](./tutorial.html) for more cool things you can do with your new toy!
|
||||
|
||||
[Join the community](http://zeppelin.apache.org/community.html), ask questions and contribute! Every little bit helps.
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import java.util.regex.Matcher;
|
|||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.elasticsearch.action.ActionResponse;
|
||||
import org.apache.zeppelin.elasticsearch.action.AggWrapper;
|
||||
import org.apache.zeppelin.elasticsearch.action.HitWrapper;
|
||||
|
|
@ -239,12 +240,13 @@ public class ElasticsearchInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String s, int i) {
|
||||
public List<InterpreterCompletion> completion(String s, int i,
|
||||
InterpreterContext interpreterContext) {
|
||||
final List suggestions = new ArrayList<>();
|
||||
|
||||
for (final String cmd : COMMANDS) {
|
||||
if (cmd.toLowerCase().contains(s)) {
|
||||
suggestions.add(new InterpreterCompletion(cmd, cmd));
|
||||
suggestions.add(new InterpreterCompletion(cmd, cmd, CompletionType.command.name()));
|
||||
}
|
||||
}
|
||||
return suggestions;
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import java.util.UUID;
|
|||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.display.AngularObjectRegistry;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
|
|
@ -305,12 +306,12 @@ public class ElasticsearchInterpreterTest {
|
|||
|
||||
@Theory
|
||||
public void testCompletion(ElasticsearchInterpreter interpreter) {
|
||||
final List<InterpreterCompletion> expectedResultOne = Arrays.asList(new InterpreterCompletion("count", "count"));
|
||||
final List<InterpreterCompletion> expectedResultTwo = Arrays.asList(new InterpreterCompletion("help", "help"));
|
||||
final List<InterpreterCompletion> expectedResultOne = Arrays.asList(new InterpreterCompletion("count", "count", CompletionType.command.name()));
|
||||
final List<InterpreterCompletion> expectedResultTwo = Arrays.asList(new InterpreterCompletion("help", "help", CompletionType.command.name()));
|
||||
|
||||
final List<InterpreterCompletion> resultOne = interpreter.completion("co", 0);
|
||||
final List<InterpreterCompletion> resultTwo = interpreter.completion("he", 0);
|
||||
final List<InterpreterCompletion> resultAll = interpreter.completion("", 0);
|
||||
final List<InterpreterCompletion> resultOne = interpreter.completion("co", 0, null);
|
||||
final List<InterpreterCompletion> resultTwo = interpreter.completion("he", 0, null);
|
||||
final List<InterpreterCompletion> resultAll = interpreter.completion("", 0, null);
|
||||
|
||||
Assert.assertEquals(expectedResultOne, resultOne);
|
||||
Assert.assertEquals(expectedResultTwo, resultTwo);
|
||||
|
|
|
|||
|
|
@ -166,7 +166,8 @@ public abstract class FileInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ import java.util.*;
|
|||
|
||||
import com.google.gson.Gson;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterException;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
|
||||
|
|
@ -247,21 +249,25 @@ public class HDFSFileInterpreter extends FileInterpreter {
|
|||
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
logger.info("Completion request at position\t" + cursor + " in string " + buf);
|
||||
final List<InterpreterCompletion> suggestions = new ArrayList<>();
|
||||
if (StringUtils.isEmpty(buf)) {
|
||||
suggestions.add(new InterpreterCompletion("ls", "ls"));
|
||||
suggestions.add(new InterpreterCompletion("cd", "cd"));
|
||||
suggestions.add(new InterpreterCompletion("pwd", "pwd"));
|
||||
suggestions.add(new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
|
||||
suggestions.add(new InterpreterCompletion("cd", "cd", CompletionType.command.name()));
|
||||
suggestions.add(new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
|
||||
return suggestions;
|
||||
}
|
||||
|
||||
//part of a command == no spaces
|
||||
if (buf.split(" ").length == 1){
|
||||
if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd"));
|
||||
if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls"));
|
||||
if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd"));
|
||||
if ("cd".contains(buf)) suggestions.add(new InterpreterCompletion("cd", "cd",
|
||||
CompletionType.command.name()));
|
||||
if ("ls".contains(buf)) suggestions.add(new InterpreterCompletion("ls", "ls",
|
||||
CompletionType.command.name()));
|
||||
if ("pwd".contains(buf)) suggestions.add(new InterpreterCompletion("pwd", "pwd",
|
||||
CompletionType.command.name()));
|
||||
|
||||
return suggestions;
|
||||
}
|
||||
|
|
@ -298,7 +304,8 @@ public class HDFSFileInterpreter extends FileInterpreter {
|
|||
String beforeLastPeriod = unfinished.substring(0, unfinished.lastIndexOf('.') + 1);
|
||||
//beforeLastPeriod should be the start of fs.pathSuffix, so take the end of it.
|
||||
String suggestedFinish = fs.pathSuffix.substring(beforeLastPeriod.length());
|
||||
suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish));
|
||||
suggestions.add(new InterpreterCompletion(suggestedFinish, suggestedFinish,
|
||||
CompletionType.path.name()));
|
||||
}
|
||||
}
|
||||
return suggestions;
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ package org.apache.zeppelin.file;
|
|||
import com.google.gson.Gson;
|
||||
import junit.framework.TestCase;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
import org.junit.Test;
|
||||
|
|
@ -106,11 +108,11 @@ public class HDFSFileInterpreterTest extends TestCase {
|
|||
|
||||
// auto completion test
|
||||
List expectedResultOne = Arrays.asList(
|
||||
new InterpreterCompletion("ls", "ls"));
|
||||
new InterpreterCompletion("ls", "ls", CompletionType.command.name()));
|
||||
List expectedResultTwo = Arrays.asList(
|
||||
new InterpreterCompletion("pwd", "pwd"));
|
||||
List<InterpreterCompletion> resultOne = t.completion("l", 0);
|
||||
List<InterpreterCompletion> resultTwo = t.completion("p", 0);
|
||||
new InterpreterCompletion("pwd", "pwd", CompletionType.command.name()));
|
||||
List<InterpreterCompletion> resultOne = t.completion("l", 0, null);
|
||||
List<InterpreterCompletion> resultTwo = t.completion("p", 0, null);
|
||||
|
||||
assertEquals(expectedResultOne, resultOne);
|
||||
assertEquals(expectedResultTwo, resultTwo);
|
||||
|
|
|
|||
|
|
@ -373,7 +373,8 @@ public class FlinkInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return new LinkedList<>();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@
|
|||
|
||||
<properties>
|
||||
<!--library versions-->
|
||||
<geode.version>1.0.0-incubating-SNAPSHOT</geode.version>
|
||||
<geode.version>1.1.0</geode.version>
|
||||
<commons.exec.version>1.3</commons.exec.version>
|
||||
</properties>
|
||||
|
||||
|
|
@ -48,7 +48,7 @@
|
|||
|
||||
<dependency>
|
||||
<groupId>org.apache.geode</groupId>
|
||||
<artifactId>gemfire-core</artifactId>
|
||||
<artifactId>geode-core</artifactId>
|
||||
<version>${geode.version}</version>
|
||||
</dependency>
|
||||
|
||||
|
|
|
|||
|
|
@ -29,12 +29,12 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.gemstone.gemfire.cache.client.ClientCache;
|
||||
import com.gemstone.gemfire.cache.client.ClientCacheFactory;
|
||||
import com.gemstone.gemfire.cache.query.QueryService;
|
||||
import com.gemstone.gemfire.cache.query.SelectResults;
|
||||
import com.gemstone.gemfire.cache.query.Struct;
|
||||
import com.gemstone.gemfire.pdx.PdxInstance;
|
||||
import org.apache.geode.cache.client.ClientCache;
|
||||
import org.apache.geode.cache.client.ClientCacheFactory;
|
||||
import org.apache.geode.cache.query.QueryService;
|
||||
import org.apache.geode.cache.query.SelectResults;
|
||||
import org.apache.geode.cache.query.Struct;
|
||||
import org.apache.geode.pdx.PdxInstance;
|
||||
|
||||
/**
|
||||
* Apache Geode OQL Interpreter (http://geode.apache.org)
|
||||
|
|
@ -282,7 +282,8 @@ public class GeodeOqlInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,14 +37,14 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
|
|||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.gemstone.gemfire.cache.query.QueryService;
|
||||
import com.gemstone.gemfire.cache.query.SelectResults;
|
||||
import com.gemstone.gemfire.cache.query.Struct;
|
||||
import com.gemstone.gemfire.cache.query.internal.StructImpl;
|
||||
import com.gemstone.gemfire.cache.query.internal.types.StructTypeImpl;
|
||||
import com.gemstone.gemfire.pdx.PdxInstance;
|
||||
import com.gemstone.gemfire.pdx.internal.PdxInstanceImpl;
|
||||
import com.gemstone.gemfire.pdx.internal.PdxType;
|
||||
import org.apache.geode.cache.query.QueryService;
|
||||
import org.apache.geode.cache.query.SelectResults;
|
||||
import org.apache.geode.cache.query.Struct;
|
||||
import org.apache.geode.cache.query.internal.StructImpl;
|
||||
import org.apache.geode.cache.query.internal.types.StructTypeImpl;
|
||||
import org.apache.geode.pdx.PdxInstance;
|
||||
import org.apache.geode.pdx.internal.PdxInstanceImpl;
|
||||
import org.apache.geode.pdx.internal.PdxType;
|
||||
|
||||
public class GeodeOqlInterpreterTest {
|
||||
|
||||
|
|
|
|||
4
groovy/README.md
Normal file
4
groovy/README.md
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
## Groovy Interpreter
|
||||
|
||||
|
||||
[see groovy documentation](../docs/interpreter/groovy.md)
|
||||
166
groovy/pom-groovy-only.xml
Normal file
166
groovy/pom-groovy-only.xml
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<artifactId>zeppelin</artifactId>
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<!--version>0.6.2</version-->
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<relativePath>..</relativePath>
|
||||
</parent>
|
||||
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<artifactId>zeppelin-groovy</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<name>Zeppelin: Groovy interpreter</name>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>zeppelin-interpreter</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.codehaus.groovy/groovy-all -->
|
||||
<dependency>
|
||||
<groupId>org.codehaus.groovy</groupId>
|
||||
<artifactId>groovy-all</artifactId>
|
||||
<version>2.4.7</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.1</version>
|
||||
<configuration>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<compilerArgs>
|
||||
<!--arg>-verbose</arg-->
|
||||
<arg>-Xlint:unchecked</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
<executions>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-enforcer-plugin</artifactId>
|
||||
<version>1.3.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>enforce</id>
|
||||
<phase>none</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>copy-artifact</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
<artifactItems>
|
||||
<artifactItem>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>${project.artifactId}</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>${project.packaging}</type>
|
||||
</artifactItem>
|
||||
</artifactItems>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<!--this one only for independent groovy interpreter assembly-->
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.5.3</version>
|
||||
<configuration>
|
||||
<descriptor>src/assembly/dep.xml</descriptor>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-archive</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
149
groovy/pom.xml
Normal file
149
groovy/pom.xml
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<artifactId>zeppelin</artifactId>
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<!--version>0.6.2</version-->
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<relativePath>..</relativePath>
|
||||
</parent>
|
||||
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
<artifactId>zeppelin-groovy</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>0.8.0-SNAPSHOT</version>
|
||||
<name>Zeppelin: Groovy interpreter</name>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>zeppelin-interpreter</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/org.codehaus.groovy/groovy-all -->
|
||||
<dependency>
|
||||
<groupId>org.codehaus.groovy</groupId>
|
||||
<artifactId>groovy-all</artifactId>
|
||||
<version>2.4.7</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.1</version>
|
||||
<configuration>
|
||||
<showDeprecation>true</showDeprecation>
|
||||
<compilerArgs>
|
||||
<!--arg>-verbose</arg-->
|
||||
<arg>-Xlint:unchecked</arg>
|
||||
</compilerArgs>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
<configuration>
|
||||
<skip>true</skip>
|
||||
</configuration>
|
||||
<executions>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-enforcer-plugin</artifactId>
|
||||
<version>1.3.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>enforce</id>
|
||||
<phase>none</phase>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-dependency-plugin</artifactId>
|
||||
<version>2.8</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-dependencies</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy-dependencies</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
</configuration>
|
||||
</execution>
|
||||
<execution>
|
||||
<id>copy-artifact</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>copy</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
|
||||
<overWriteReleases>false</overWriteReleases>
|
||||
<overWriteSnapshots>false</overWriteSnapshots>
|
||||
<overWriteIfNewer>true</overWriteIfNewer>
|
||||
<includeScope>runtime</includeScope>
|
||||
<artifactItems>
|
||||
<artifactItem>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>${project.artifactId}</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<type>${project.packaging}</type>
|
||||
</artifactItem>
|
||||
</artifactItems>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
66
groovy/src/assembly/dep.xml
Normal file
66
groovy/src/assembly/dep.xml
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
|
||||
<id>bin</id>
|
||||
<baseDirectory>groovy</baseDirectory>
|
||||
<formats>
|
||||
<format>zip</format>
|
||||
</formats>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>${project.basedir}</directory>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<filtered>true</filtered>
|
||||
<includes>
|
||||
<include>README*</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.build.directory}</directory>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<includes>
|
||||
<include>*.jar</include>
|
||||
<include>revision.txt</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<!--fileSet>
|
||||
<directory>${project.basedir}/src/main/groovy/classes/</directory>
|
||||
<outputDirectory>/classes/</outputDirectory>
|
||||
<includes>
|
||||
<include>*.groovy</include>
|
||||
</includes>
|
||||
</fileSet-->
|
||||
|
||||
<!--fileSet>
|
||||
<directory>${project.build.directory}/site</directory>
|
||||
<outputDirectory>docs</outputDirectory>
|
||||
</fileSet-->
|
||||
</fileSets>
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<unpack>false</unpack>
|
||||
<scope>runtime</scope>
|
||||
<!--excludes>
|
||||
<exclude>junit:junit</exclude>
|
||||
</excludes-->
|
||||
</dependencySet>
|
||||
</dependencySets>
|
||||
</assembly>
|
||||
1
groovy/src/assembly/readme.txt
Normal file
1
groovy/src/assembly/readme.txt
Normal file
|
|
@ -0,0 +1 @@
|
|||
to assemble groovy interpreter separately
|
||||
369
groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java
Normal file
369
groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.zeppelin.groovy;
|
||||
|
||||
|
||||
import java.io.StringWriter;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.Properties;
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.List;
|
||||
import java.util.LinkedList;
|
||||
|
||||
import groovy.xml.MarkupBuilder;
|
||||
import groovy.lang.Closure;
|
||||
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
|
||||
|
||||
import org.apache.zeppelin.display.AngularObjectRegistry;
|
||||
import org.apache.zeppelin.display.AngularObject;
|
||||
import org.apache.zeppelin.display.GUI;
|
||||
import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
|
||||
import org.apache.zeppelin.annotation.ZeppelinApi;
|
||||
import org.apache.zeppelin.interpreter.RemoteWorksController;
|
||||
import org.apache.zeppelin.interpreter.InterpreterException;
|
||||
|
||||
/**
|
||||
* Groovy interpreter for Zeppelin.
|
||||
*/
|
||||
public class GObject extends groovy.lang.GroovyObjectSupport {
|
||||
|
||||
Logger log;
|
||||
StringWriter out;
|
||||
Properties props;
|
||||
InterpreterContext interpreterContext;
|
||||
Map<String, Object> bindings;
|
||||
|
||||
|
||||
public GObject(Logger log, StringWriter out, Properties p, InterpreterContext ctx,
|
||||
Map<String, Object> bindings) {
|
||||
this.log = log;
|
||||
this.out = out;
|
||||
this.interpreterContext = ctx;
|
||||
this.props = p;
|
||||
this.bindings = bindings;
|
||||
}
|
||||
|
||||
public Object getProperty(String key) {
|
||||
if ("log".equals(key)) {
|
||||
return log;
|
||||
}
|
||||
return props.getProperty(key);
|
||||
}
|
||||
|
||||
public void setProperty(String key, Object value) {
|
||||
throw new RuntimeException("Set properties not supported: " + key + "=" + value);
|
||||
}
|
||||
|
||||
public Properties getProperties() {
|
||||
return props;
|
||||
}
|
||||
|
||||
private void startOutputType(String type) {
|
||||
StringBuffer sb = out.getBuffer();
|
||||
if (sb.length() > 0) {
|
||||
if (sb.length() < type.length() || !type.equals(sb.substring(0, type.length()))) {
|
||||
log.error("try to start output `" + type + "` after non-" + type + " started");
|
||||
}
|
||||
} else {
|
||||
out.append(type);
|
||||
out.append('\n');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* returns gui object
|
||||
*/
|
||||
public GUI getGui() {
|
||||
return interpreterContext.getGui();
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object input(String name) {
|
||||
return input(name, "");
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object input(String name, Object defaultValue) {
|
||||
return getGui().input(name, defaultValue);
|
||||
}
|
||||
|
||||
private ParamOption[] toParamOptions(Map<Object, String> options) {
|
||||
ParamOption[] paramOptions = new ParamOption[options.size()];
|
||||
int i = 0;
|
||||
for (Map.Entry<Object, String> e : options.entrySet()) {
|
||||
paramOptions[i++] = new ParamOption(e.getKey(), e.getValue());
|
||||
}
|
||||
return paramOptions;
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object select(String name, Map<Object, String> options) {
|
||||
return select(name, "", options);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object select(String name, Object defaultValue, Map<Object, String> options) {
|
||||
return getGui().select(name, defaultValue, toParamOptions(options));
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Collection<Object> checkbox(String name, Map<Object, String> options) {
|
||||
return checkbox(name, options.keySet(), options);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Collection<Object> checkbox(String name, Collection<Object> defaultChecked,
|
||||
Map<Object, String> options) {
|
||||
return getGui().checkbox(name, defaultChecked, toParamOptions(options));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns shared variable if it was previously set. The same as getting groovy script variables
|
||||
* but this method will return null if script variable not assigned. To understand groovy script
|
||||
* variables see groovy.transform.Field annotation for more information.
|
||||
*
|
||||
* @see #put
|
||||
*/
|
||||
public Object get(String varName) {
|
||||
return bindings.get(varName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns script (shared) variable value but if value was not set returns default value. The same
|
||||
* as getting groovy script variables but this method will return default value if script variable
|
||||
* not assigned. To understand groovy script variables see groovy.transform.Field annotation for
|
||||
* more information.
|
||||
*
|
||||
* @see #put
|
||||
*/
|
||||
public Object get(String varName, Object defValue) {
|
||||
return bindings.containsKey(varName) ? bindings.get(varName) : defValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a new value to interpreter's shared variables.
|
||||
* Could be set by <code>put('varName', newValue )</code>
|
||||
* or by just assigning <code>varName = value</code> without declaring a variable.
|
||||
*/
|
||||
public Object put(String varName, Object newValue) {
|
||||
return bindings.put(varName, newValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* starts or continues rendering html/angular and returns MarkupBuilder to build html.
|
||||
* <pre> g.html().with{
|
||||
* h1("hello")
|
||||
* h2("world")
|
||||
* }</pre>
|
||||
*/
|
||||
public MarkupBuilder html() {
|
||||
startOutputType("%angular");
|
||||
return new MarkupBuilder(out);
|
||||
}
|
||||
|
||||
/**
|
||||
* starts or continues rendering table rows
|
||||
*
|
||||
* @param obj: 1. List(rows) of List(columns) where first line is a header
|
||||
*/
|
||||
public void table(Object obj) {
|
||||
if (obj == null) {
|
||||
return;
|
||||
}
|
||||
StringBuffer sb = out.getBuffer();
|
||||
startOutputType("%table");
|
||||
if (obj instanceof groovy.lang.Closure) {
|
||||
//if closure run and get result collection
|
||||
obj = ((Closure) obj).call();
|
||||
}
|
||||
if (obj instanceof Collection) {
|
||||
int count = 0;
|
||||
for (Object row : ((Collection) obj)) {
|
||||
count++;
|
||||
boolean rowStarted = false;
|
||||
if (row instanceof Collection) {
|
||||
for (Object field : ((Collection) row)) {
|
||||
if (rowStarted) {
|
||||
sb.append('\t');
|
||||
}
|
||||
sb.append(field);
|
||||
rowStarted = true;
|
||||
}
|
||||
} else {
|
||||
sb.append(row);
|
||||
}
|
||||
sb.append('\n');
|
||||
}
|
||||
} else {
|
||||
throw new RuntimeException("Not supported table value :" + obj.getClass());
|
||||
}
|
||||
}
|
||||
|
||||
private AngularObject getAngularObject(String name) {
|
||||
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
|
||||
String noteId = interpreterContext.getNoteId();
|
||||
// try get local object
|
||||
AngularObject paragraphAo = registry.get(name, noteId, interpreterContext.getParagraphId());
|
||||
AngularObject noteAo = registry.get(name, noteId, null);
|
||||
|
||||
AngularObject ao = paragraphAo != null ? paragraphAo : noteAo;
|
||||
|
||||
if (ao == null) {
|
||||
// then global object
|
||||
ao = registry.get(name, null, null);
|
||||
}
|
||||
return ao;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get angular object. Look up notebook scope first and then global scope
|
||||
*
|
||||
* @param name variable name
|
||||
* @return value
|
||||
*/
|
||||
public Object angular(String name) {
|
||||
AngularObject ao = getAngularObject(name);
|
||||
if (ao == null) {
|
||||
return null;
|
||||
} else {
|
||||
return ao.get();
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void angularBind(String name, Object o, String noteId) {
|
||||
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
|
||||
|
||||
if (registry.get(name, noteId, null) == null) {
|
||||
registry.add(name, o, noteId, null);
|
||||
} else {
|
||||
registry.get(name, noteId, null).set(o);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create angular variable in notebook scope and bind with front end Angular display system.
|
||||
* If variable exists, it'll be overwritten.
|
||||
*
|
||||
* @param name name of the variable
|
||||
* @param o value
|
||||
*/
|
||||
public void angularBind(String name, Object o) {
|
||||
angularBind(name, o, interpreterContext.getNoteId());
|
||||
}
|
||||
|
||||
/*------------------------------------------RUN----------------------------------------*/
|
||||
@ZeppelinApi
|
||||
public List<InterpreterContextRunner> getInterpreterContextRunner(String noteId,
|
||||
String paragraphId, InterpreterContext interpreterContext) {
|
||||
RemoteWorksController remoteWorksController = interpreterContext.getRemoteWorksController();
|
||||
if (remoteWorksController != null) {
|
||||
return remoteWorksController.getRemoteContextRunner(noteId, paragraphId);
|
||||
}
|
||||
return new LinkedList<InterpreterContextRunner>();
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public List<InterpreterContextRunner> getInterpreterContextRunner(String noteId,
|
||||
InterpreterContext interpreterContext) {
|
||||
RemoteWorksController remoteWorksController = interpreterContext.getRemoteWorksController();
|
||||
if (remoteWorksController != null) {
|
||||
return remoteWorksController.getRemoteContextRunner(noteId);
|
||||
}
|
||||
return new LinkedList<InterpreterContextRunner>();
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph by id
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String noteId, String paragraphId) {
|
||||
run(noteId, paragraphId, interpreterContext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph by id
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String paragraphId) {
|
||||
String noteId = interpreterContext.getNoteId();
|
||||
run(noteId, paragraphId, interpreterContext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph by id
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String noteId, String paragraphId, InterpreterContext context) {
|
||||
if (paragraphId.equals(context.getParagraphId())) {
|
||||
throw new InterpreterException("Can not run current Paragraph");
|
||||
}
|
||||
List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, paragraphId,
|
||||
context);
|
||||
if (runners.size() <= 0) {
|
||||
throw new InterpreterException("Paragraph " + paragraphId + " not found " + runners.size());
|
||||
}
|
||||
for (InterpreterContextRunner r : runners) {
|
||||
r.run();
|
||||
}
|
||||
}
|
||||
|
||||
public void runNote(String noteId) {
|
||||
runNote(noteId, interpreterContext);
|
||||
}
|
||||
|
||||
public void runNote(String noteId, InterpreterContext context) {
|
||||
String runningNoteId = context.getNoteId();
|
||||
String runningParagraphId = context.getParagraphId();
|
||||
List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
|
||||
|
||||
if (runners.size() <= 0) {
|
||||
throw new InterpreterException("Note " + noteId + " not found " + runners.size());
|
||||
}
|
||||
|
||||
for (InterpreterContextRunner r : runners) {
|
||||
if (r.getNoteId().equals(runningNoteId) && r.getParagraphId().equals(runningParagraphId)) {
|
||||
continue;
|
||||
}
|
||||
r.run();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all paragraphs. except this.
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void runAll() {
|
||||
runAll(interpreterContext);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run all paragraphs. except this.
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void runAll(InterpreterContext context) {
|
||||
runNote(context.getNoteId());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,216 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.groovy;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.File;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
import org.apache.zeppelin.scheduler.Job;
|
||||
import org.apache.zeppelin.scheduler.Scheduler;
|
||||
import org.apache.zeppelin.scheduler.SchedulerFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import groovy.lang.GroovyShell;
|
||||
import groovy.lang.Script;
|
||||
import org.codehaus.groovy.control.CompilerConfiguration;
|
||||
import org.codehaus.groovy.runtime.ResourceGroovyMethods;
|
||||
import org.codehaus.groovy.runtime.StackTraceUtils;
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Groovy interpreter for Zeppelin.
|
||||
*/
|
||||
public class GroovyInterpreter extends Interpreter {
|
||||
|
||||
Logger log = LoggerFactory.getLogger(GroovyInterpreter.class);
|
||||
GroovyShell shell = null; //new GroovyShell();
|
||||
//here we will store Interpreters shared variables. concurrent just in case.
|
||||
Map<String, Object> sharedBindings = new ConcurrentHashMap<String, Object>();
|
||||
//cache for groovy compiled scripts
|
||||
Map<String, Class<Script>> scriptCache = Collections
|
||||
.synchronizedMap(new WeakHashMap<String, Class<Script>>(100));
|
||||
|
||||
|
||||
public GroovyInterpreter(Properties property) {
|
||||
super(property);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void open() {
|
||||
CompilerConfiguration conf = new CompilerConfiguration();
|
||||
conf.setDebug(true);
|
||||
shell = new GroovyShell(conf);
|
||||
String classes = getProperty("GROOVY_CLASSES");
|
||||
if (classes == null || classes.length() == 0) {
|
||||
try {
|
||||
File jar = new File(
|
||||
GroovyInterpreter.class.getProtectionDomain().getCodeSource().getLocation().toURI()
|
||||
.getPath());
|
||||
classes = new File(jar.getParentFile(), "classes").toString();
|
||||
} catch (Exception e) {
|
||||
}
|
||||
}
|
||||
log.info("groovy classes classpath: " + classes);
|
||||
if (classes != null && classes.length() > 0) {
|
||||
File fClasses = new File(classes);
|
||||
if (!fClasses.exists()) {
|
||||
fClasses.mkdirs();
|
||||
}
|
||||
shell.getClassLoader().addClasspath(classes);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
shell = null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FormType getFormType() {
|
||||
return FormType.NATIVE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getProgress(InterpreterContext context) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Scheduler getScheduler() {
|
||||
return SchedulerFactory.singleton()
|
||||
.createOrGetParallelScheduler(GroovyInterpreter.class.getName() + this.hashCode(), 10);
|
||||
}
|
||||
|
||||
private Job getRunningJob(String paragraphId) {
|
||||
Job foundJob = null;
|
||||
Collection<Job> jobsRunning = getScheduler().getJobsRunning();
|
||||
for (Job job : jobsRunning) {
|
||||
if (job.getId().equals(paragraphId)) {
|
||||
foundJob = job;
|
||||
}
|
||||
}
|
||||
return foundJob;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Script getGroovyScript(String id, String scriptText) /*throws SQLException*/ {
|
||||
if (shell == null) {
|
||||
throw new RuntimeException("Groovy Shell is not initialized: null");
|
||||
}
|
||||
try {
|
||||
Class<Script> clazz = scriptCache.get(scriptText);
|
||||
if (clazz == null) {
|
||||
String scriptName = id + "_" + Long.toHexString(scriptText.hashCode()) + ".groovy";
|
||||
clazz = (Class<Script>) shell.parse(scriptText, scriptName).getClass();
|
||||
scriptCache.put(scriptText, clazz);
|
||||
}
|
||||
|
||||
Script script = (Script) clazz.newInstance();
|
||||
return script;
|
||||
} catch (Throwable t) {
|
||||
throw new RuntimeException("Failed to parse groovy script: " + t, t);
|
||||
}
|
||||
}
|
||||
|
||||
private static Set<String> predefinedBindings = new HashSet<String>();
|
||||
|
||||
static {
|
||||
predefinedBindings.add("g");
|
||||
predefinedBindings.add("out");
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
|
||||
try {
|
||||
Script script = getGroovyScript(contextInterpreter.getParagraphId(), cmd);
|
||||
Job runningJob = getRunningJob(contextInterpreter.getParagraphId());
|
||||
runningJob.info()
|
||||
.put("CURRENT_THREAD", Thread.currentThread()); //to be able to terminate thread
|
||||
Map<String, Object> bindings = script.getBinding().getVariables();
|
||||
bindings.clear();
|
||||
StringWriter out = new StringWriter((int) (cmd.length() * 1.75));
|
||||
//put shared bindings evaluated in this interpreter
|
||||
bindings.putAll(sharedBindings);
|
||||
//put predefined bindings
|
||||
bindings.put("g", new GObject(log, out, this.getProperty(), contextInterpreter, bindings));
|
||||
bindings.put("out", new PrintWriter(out, true));
|
||||
|
||||
script.run();
|
||||
//let's get shared variables defined in current script and store them in shared map
|
||||
for (Map.Entry<String, Object> e : bindings.entrySet()) {
|
||||
if (!predefinedBindings.contains(e.getKey())) {
|
||||
if (log.isTraceEnabled()) {
|
||||
log.trace("groovy script variable " + e); //let's see what we have...
|
||||
}
|
||||
sharedBindings.put(e.getKey(), e.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
bindings.clear();
|
||||
InterpreterResult result = new InterpreterResult(Code.SUCCESS, out.toString());
|
||||
return result;
|
||||
} catch (Throwable t) {
|
||||
t = StackTraceUtils.deepSanitize(t);
|
||||
String msg = t.toString() + "\n at " + t.getStackTrace()[0];
|
||||
log.error("Failed to run script: " + t + "\n" + cmd + "\n", t);
|
||||
return new InterpreterResult(Code.ERROR, msg);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void cancel(InterpreterContext context) {
|
||||
Job runningJob = getRunningJob(context.getParagraphId());
|
||||
if (runningJob != null) {
|
||||
Map<String, Object> info = runningJob.info();
|
||||
Object object = info.get("CURRENT_THREAD");
|
||||
if (object instanceof Thread) {
|
||||
try {
|
||||
Thread t = (Thread) object;
|
||||
t.dumpStack();
|
||||
t.interrupt();
|
||||
//t.stop(); //TODO: need some way to terminate maybe through GObject..
|
||||
} catch (Throwable t) {
|
||||
log.error("Failed to cancel script: " + t, t);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
154
groovy/src/main/resources/HTTP.groovy
Normal file
154
groovy/src/main/resources/HTTP.groovy
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import groovy.json.JsonOutput
|
||||
|
||||
/**
|
||||
* simple http rest client for groovy
|
||||
* by dlukyanov@ukr.net
|
||||
*/
|
||||
@groovy.transform.CompileStatic
|
||||
public class HTTP{
|
||||
//default response handler
|
||||
public static Closure TEXT_RECEIVER = {InputStream instr,Map ctx->
|
||||
return instr.getText( (String)ctx.encoding );
|
||||
}
|
||||
|
||||
public static Closure JSON_RECEIVER = { InputStream instr, Map ctx->
|
||||
return new groovy.json.JsonSlurper().parse(instr,(String)ctx.encoding);
|
||||
}
|
||||
|
||||
public static Closure FILE_RECEIVER(File f){
|
||||
return { InputStream instr, Map ctx->
|
||||
f<<instr;
|
||||
return f;
|
||||
}
|
||||
}
|
||||
|
||||
public static Map<String,Object> get(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','GET');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> post(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','POST');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> put(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','PUT');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> delete(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','DELETE');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> send(Map<String,Object> ctx)throws IOException{
|
||||
String url = ctx.url;
|
||||
Map<String,String> headers = (Map<String,String>)ctx.headers;
|
||||
String method = ctx.method;
|
||||
Object body = ctx.body;
|
||||
String encoding = ctx.encoding?:"UTF-8";
|
||||
Closure receiver = (Closure)ctx.receiver;
|
||||
Map<String,String> query = (Map<String,String>)ctx.query;
|
||||
|
||||
//copy context and set default values
|
||||
ctx = [:] + ctx;
|
||||
ctx.encoding = encoding;
|
||||
String contentType="";
|
||||
|
||||
if(query){
|
||||
url+="?"+query.collect{k,v-> k+"="+URLEncoder.encode(v,'UTF-8') }.join('&')
|
||||
}
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
||||
|
||||
connection.setDoOutput(true);
|
||||
connection.setRequestMethod(method);
|
||||
if ( headers!=null && !headers.isEmpty() ) {
|
||||
//add headers
|
||||
for (Map.Entry<String, String> entry : headers.entrySet()) {
|
||||
connection.addRequestProperty(entry.getKey(), entry.getValue());
|
||||
if("content-type".equals(entry.getKey().toLowerCase()))contentType=entry.getValue();
|
||||
}
|
||||
}
|
||||
|
||||
if(body!=null){
|
||||
//write body
|
||||
OutputStream out = connection.getOutputStream();
|
||||
if( body instanceof Closure ){
|
||||
((Closure)body).call(out, ctx);
|
||||
}else if(body instanceof InputStream){
|
||||
out << (InputStream)body;
|
||||
}else if(body instanceof Map){
|
||||
if( contentType.matches("(?i)[^/]+/json") ){
|
||||
out.withWriter((String)ctx.encoding){
|
||||
it.append( JsonOutput.toJson((Map)body) );
|
||||
it.flush();
|
||||
}
|
||||
}else{
|
||||
throw new IOException("Map body type supported only for */json content-type");
|
||||
}
|
||||
}else if(body instanceof CharSequence){
|
||||
out.withWriter((String)ctx.encoding){
|
||||
it.append((CharSequence)body);
|
||||
it.flush();
|
||||
}
|
||||
}else{
|
||||
throw new IOException("Unsupported body type: "+body.getClass());
|
||||
}
|
||||
out.flush();
|
||||
out.close();
|
||||
out=null;
|
||||
}
|
||||
|
||||
Map response = [:];
|
||||
ctx.response = response;
|
||||
response.code = connection.getResponseCode();
|
||||
response.message = connection.getResponseMessage();
|
||||
response.headers = connection.getHeaderFields();
|
||||
|
||||
InputStream instr = null;
|
||||
|
||||
if( ((int)response.code)>=400 ){
|
||||
try{
|
||||
instr = connection.getErrorStream();
|
||||
}catch(Exception ei){}
|
||||
}else{
|
||||
try{
|
||||
instr = connection.getInputStream();
|
||||
}catch(java.io.IOException ei){
|
||||
throw new IOException("fail to open InputStream for http code "+response.code+":"+ei);
|
||||
}
|
||||
}
|
||||
|
||||
if(instr!=null) {
|
||||
instr = new BufferedInputStream(instr);
|
||||
if(receiver==null){
|
||||
if( response.headers['Content-Type']?.toString()?.indexOf('/json')>0 ){
|
||||
receiver=JSON_RECEIVER;
|
||||
} else receiver=TEXT_RECEIVER;
|
||||
}
|
||||
response.body = receiver(instr,ctx);
|
||||
instr.close();
|
||||
instr=null;
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
}
|
||||
15
groovy/src/main/resources/interpreter-setting.json
Normal file
15
groovy/src/main/resources/interpreter-setting.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[
|
||||
{
|
||||
"group": "groovy",
|
||||
"name": "groovy",
|
||||
"className": "org.apache.zeppelin.groovy.GroovyInterpreter",
|
||||
"properties": {
|
||||
"GROOVY_CLASSES": {
|
||||
"envName": null,
|
||||
"propertyName": "GROOVY_CLASSES",
|
||||
"defaultValue": "",
|
||||
"description": "The path for custom groovy classes location. If empty `./interpreter/groovy/classes`"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -145,7 +145,8 @@ public class HbaseInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -98,7 +98,8 @@ public class DevInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return new LinkedList<>();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -331,7 +331,8 @@ public class IgniteInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return new LinkedList<>();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -184,7 +184,8 @@ public class IgniteSqlInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return new LinkedList<>();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
12
jdbc/pom.xml
12
jdbc/pom.xml
|
|
@ -35,7 +35,6 @@
|
|||
<properties>
|
||||
<!--library versions-->
|
||||
<postgresql.version>9.4-1201-jdbc41</postgresql.version>
|
||||
<jline.version>2.12.1</jline.version>
|
||||
<hadoop.common.version>2.7.2</hadoop.common.version>
|
||||
<h2.version>1.4.190</h2.version>
|
||||
<commons.dbcp2.version>2.0.1</commons.dbcp2.version>
|
||||
|
|
@ -68,17 +67,6 @@
|
|||
<artifactId>slf4j-log4j12</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>jline</groupId>
|
||||
<artifactId>jline</artifactId>
|
||||
<version>${jline.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
<artifactId>h2</artifactId>
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import org.apache.zeppelin.interpreter.InterpreterContext;
|
|||
import org.apache.zeppelin.interpreter.InterpreterException;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
|
||||
import org.apache.zeppelin.interpreter.ResultMessages;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
import org.apache.zeppelin.jdbc.security.JDBCSecurityImpl;
|
||||
import org.apache.zeppelin.scheduler.Scheduler;
|
||||
|
|
@ -56,9 +57,7 @@ import org.apache.zeppelin.user.UsernamePassword;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.Lists;
|
||||
|
||||
import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
|
||||
import static org.apache.commons.lang.StringUtils.isEmpty;
|
||||
|
|
@ -102,6 +101,7 @@ public class JDBCInterpreter extends Interpreter {
|
|||
static final String USER_KEY = "user";
|
||||
static final String PASSWORD_KEY = "password";
|
||||
static final String PRECODE_KEY = "precode";
|
||||
static final String COMPLETER_SCHEMA_FILTERS_KEY = "completer.schemaFilters";
|
||||
static final String JDBC_JCEKS_FILE = "jceks.file";
|
||||
static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey";
|
||||
static final String PRECODE_KEY_TEMPLATE = "%s.precode";
|
||||
|
|
@ -129,22 +129,12 @@ public class JDBCInterpreter extends Interpreter {
|
|||
|
||||
private final HashMap<String, Properties> basePropretiesMap;
|
||||
private final HashMap<String, JDBCUserConfigurations> jdbcUserConfigurationsMap;
|
||||
private final Map<String, SqlCompleter> propertyKeySqlCompleterMap;
|
||||
|
||||
private static final Function<CharSequence, InterpreterCompletion> sequenceToStringTransformer =
|
||||
new Function<CharSequence, InterpreterCompletion>() {
|
||||
public InterpreterCompletion apply(CharSequence seq) {
|
||||
return new InterpreterCompletion(seq.toString(), seq.toString());
|
||||
}
|
||||
};
|
||||
|
||||
private static final List<InterpreterCompletion> NO_COMPLETION = new ArrayList<>();
|
||||
private int maxLineResults;
|
||||
|
||||
public JDBCInterpreter(Properties property) {
|
||||
super(property);
|
||||
jdbcUserConfigurationsMap = new HashMap<>();
|
||||
propertyKeySqlCompleterMap = new HashMap<>();
|
||||
basePropretiesMap = new HashMap<>();
|
||||
maxLineResults = MAX_LINE_DEFAULT;
|
||||
}
|
||||
|
|
@ -159,7 +149,7 @@ public class JDBCInterpreter extends Interpreter {
|
|||
logger.debug("propertyKey: {}", propertyKey);
|
||||
String[] keyValue = propertyKey.split("\\.", 2);
|
||||
if (2 == keyValue.length) {
|
||||
logger.info("key: {}, value: {}", keyValue[0], keyValue[1]);
|
||||
logger.debug("key: {}, value: {}", keyValue[0], keyValue[1]);
|
||||
|
||||
Properties prefixProperties;
|
||||
if (basePropretiesMap.containsKey(keyValue[0])) {
|
||||
|
|
@ -192,9 +182,7 @@ public class JDBCInterpreter extends Interpreter {
|
|||
if (!isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
|
||||
JDBCSecurityImpl.createSecureConfiguration(property);
|
||||
}
|
||||
for (String propertyKey : basePropretiesMap.keySet()) {
|
||||
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(null));
|
||||
}
|
||||
|
||||
setMaxLineResults();
|
||||
}
|
||||
|
||||
|
|
@ -205,10 +193,11 @@ public class JDBCInterpreter extends Interpreter {
|
|||
}
|
||||
}
|
||||
|
||||
private SqlCompleter createSqlCompleter(Connection jdbcConnection) {
|
||||
|
||||
private SqlCompleter createSqlCompleter(Connection jdbcConnection, String propertyKey) {
|
||||
String schemaFiltersKey = String.format("%s.%s", propertyKey, COMPLETER_SCHEMA_FILTERS_KEY);
|
||||
String filters = getProperty(schemaFiltersKey);
|
||||
SqlCompleter completer = new SqlCompleter();
|
||||
completer.initFromConnection(jdbcConnection, "");
|
||||
completer.initFromConnection(jdbcConnection, filters);
|
||||
return completer;
|
||||
}
|
||||
|
||||
|
|
@ -260,6 +249,7 @@ public class JDBCInterpreter extends Interpreter {
|
|||
|
||||
private boolean existAccountInBaseProperty(String propertyKey) {
|
||||
return basePropretiesMap.get(propertyKey).containsKey(USER_KEY) &&
|
||||
!isEmpty((String) basePropretiesMap.get(propertyKey).get(USER_KEY)) &&
|
||||
basePropretiesMap.get(propertyKey).containsKey(PASSWORD_KEY);
|
||||
}
|
||||
|
||||
|
|
@ -306,7 +296,6 @@ public class JDBCInterpreter extends Interpreter {
|
|||
}
|
||||
}
|
||||
jdbcUserConfigurations.setPropertyMap(propertyKey, basePropretiesMap.get(propertyKey));
|
||||
|
||||
if (existAccountInBaseProperty(propertyKey)) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -368,7 +357,8 @@ public class JDBCInterpreter extends Interpreter {
|
|||
|
||||
switch (authType) {
|
||||
case KERBEROS:
|
||||
if (user == null) {
|
||||
if (user == null || "false".equalsIgnoreCase(
|
||||
property.getProperty("zeppelin.jdbc.auth.kerberos.proxy.enable"))) {
|
||||
connection = getConnectionFromPool(url, user, propertyKey, properties);
|
||||
} else {
|
||||
if (url.trim().startsWith("jdbc:hive")) {
|
||||
|
|
@ -420,7 +410,7 @@ public class JDBCInterpreter extends Interpreter {
|
|||
connection = getConnectionFromPool(url, user, propertyKey, properties);
|
||||
}
|
||||
}
|
||||
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(connection));
|
||||
|
||||
return connection;
|
||||
}
|
||||
|
||||
|
|
@ -471,7 +461,7 @@ public class JDBCInterpreter extends Interpreter {
|
|||
msg.append(NEWLINE);
|
||||
|
||||
int displayRowCount = 0;
|
||||
while (resultSet.next() && displayRowCount < getMaxResult()) {
|
||||
while (displayRowCount < getMaxResult() && resultSet.next()) {
|
||||
for (int i = 1; i < md.getColumnCount() + 1; i++) {
|
||||
Object resultObject;
|
||||
String resultValue;
|
||||
|
|
@ -503,19 +493,36 @@ public class JDBCInterpreter extends Interpreter {
|
|||
protected ArrayList<String> splitSqlQueries(String sql) {
|
||||
ArrayList<String> queries = new ArrayList<>();
|
||||
StringBuilder query = new StringBuilder();
|
||||
Character character;
|
||||
char character;
|
||||
|
||||
Boolean antiSlash = false;
|
||||
Boolean multiLineComment = false;
|
||||
Boolean singleLineComment = false;
|
||||
Boolean quoteString = false;
|
||||
Boolean doubleQuoteString = false;
|
||||
|
||||
for (int item = 0; item < sql.length(); item++) {
|
||||
character = sql.charAt(item);
|
||||
|
||||
if (character.equals('\\')) {
|
||||
if ((singleLineComment && (character == '\n' || item == sql.length() - 1))
|
||||
|| (multiLineComment && character == '/' && sql.charAt(item - 1) == '*')) {
|
||||
singleLineComment = false;
|
||||
multiLineComment = false;
|
||||
if (item == sql.length() - 1 && query.length() > 0) {
|
||||
queries.add(StringUtils.trim(query.toString()));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (singleLineComment || multiLineComment) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (character == '\\') {
|
||||
antiSlash = true;
|
||||
}
|
||||
if (character.equals('\'')) {
|
||||
|
||||
if (character == '\'') {
|
||||
if (antiSlash) {
|
||||
antiSlash = false;
|
||||
} else if (quoteString) {
|
||||
|
|
@ -524,7 +531,8 @@ public class JDBCInterpreter extends Interpreter {
|
|||
quoteString = true;
|
||||
}
|
||||
}
|
||||
if (character.equals('"')) {
|
||||
|
||||
if (character == '"') {
|
||||
if (antiSlash) {
|
||||
antiSlash = false;
|
||||
} else if (doubleQuoteString) {
|
||||
|
|
@ -534,16 +542,30 @@ public class JDBCInterpreter extends Interpreter {
|
|||
}
|
||||
}
|
||||
|
||||
if (character.equals(';') && !antiSlash && !quoteString && !doubleQuoteString) {
|
||||
queries.add(query.toString());
|
||||
if (!quoteString && !doubleQuoteString && !multiLineComment && !singleLineComment
|
||||
&& sql.length() > item + 1) {
|
||||
if (character == '-' && sql.charAt(item + 1) == '-') {
|
||||
singleLineComment = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (character == '/' && sql.charAt(item + 1) == '*') {
|
||||
multiLineComment = true;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (character == ';' && !antiSlash && !quoteString && !doubleQuoteString) {
|
||||
queries.add(StringUtils.trim(query.toString()));
|
||||
query = new StringBuilder();
|
||||
} else if (item == sql.length() - 1) {
|
||||
query.append(character);
|
||||
queries.add(query.toString());
|
||||
queries.add(StringUtils.trim(query.toString()));
|
||||
} else {
|
||||
query.append(character);
|
||||
}
|
||||
}
|
||||
|
||||
return queries;
|
||||
}
|
||||
|
||||
|
|
@ -600,8 +622,13 @@ public class JDBCInterpreter extends Interpreter {
|
|||
interpreterResult.add(InterpreterResult.Type.TEXT,
|
||||
"Query executed successfully.");
|
||||
} else {
|
||||
interpreterResult.add(
|
||||
getResults(resultSet, !containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE)));
|
||||
String results = getResults(resultSet,
|
||||
!containsIgnoreCase(sqlToExecute, EXPLAIN_PREDICATE));
|
||||
interpreterResult.add(results);
|
||||
if (resultSet.next()) {
|
||||
interpreterResult.add(ResultMessages.getExceedsLimitRowsMessage(getMaxResult(),
|
||||
String.format("%s.%s", COMMON_KEY, MAX_LINE_KEY)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Response contains either an update count or there are no results.
|
||||
|
|
@ -691,7 +718,7 @@ public class JDBCInterpreter extends Interpreter {
|
|||
|
||||
@Override
|
||||
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
|
||||
logger.info("Run SQL command '{}'", cmd);
|
||||
logger.debug("Run SQL command '{}'", cmd);
|
||||
String propertyKey = getPropertyKey(cmd);
|
||||
|
||||
if (null != propertyKey && !propertyKey.equals(DEFAULT_KEY)) {
|
||||
|
|
@ -699,8 +726,7 @@ public class JDBCInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
cmd = cmd.trim();
|
||||
|
||||
logger.info("PropertyKey: {}, SQL command: '{}'", propertyKey, cmd);
|
||||
logger.debug("PropertyKey: {}, SQL command: '{}'", propertyKey, cmd);
|
||||
return executeSql(propertyKey, cmd, contextInterpreter);
|
||||
}
|
||||
|
||||
|
|
@ -753,18 +779,26 @@ public class JDBCInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
List<CharSequence> candidates = new ArrayList<>();
|
||||
SqlCompleter sqlCompleter = propertyKeySqlCompleterMap.get(getPropertyKey(buf));
|
||||
// It's strange but here cursor comes with additional +1 (even if buf is "" cursor = 1)
|
||||
if (sqlCompleter != null && sqlCompleter.complete(buf, cursor - 1, candidates) >= 0) {
|
||||
List<InterpreterCompletion> completion;
|
||||
completion = Lists.transform(candidates, sequenceToStringTransformer);
|
||||
|
||||
return completion;
|
||||
} else {
|
||||
return NO_COMPLETION;
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
List<InterpreterCompletion> candidates = new ArrayList<>();
|
||||
String propertyKey = getPropertyKey(buf);
|
||||
Connection connection = null;
|
||||
try {
|
||||
if (interpreterContext != null) {
|
||||
connection = getConnection(propertyKey, interpreterContext);
|
||||
}
|
||||
} catch (ClassNotFoundException | SQLException | IOException e) {
|
||||
logger.warn("SQLCompleter will created without use connection");
|
||||
}
|
||||
|
||||
SqlCompleter sqlCompleter = createSqlCompleter(connection, propertyKey);
|
||||
|
||||
if (sqlCompleter != null) {
|
||||
sqlCompleter.complete(buf, cursor - 1, candidates);
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
public int getMaxResult() {
|
||||
|
|
|
|||
|
|
@ -4,15 +4,6 @@ package org.apache.zeppelin.jdbc;
|
|||
* This source file is based on code taken from SQLLine 1.0.2 See SQLLine notice in LICENSE
|
||||
*/
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.common.collect.Sets.SetView;
|
||||
import jline.console.completer.ArgumentCompleter.ArgumentList;
|
||||
import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
|
||||
import jline.console.completer.StringsCompleter;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
|
|
@ -20,15 +11,34 @@ import java.sql.Connection;
|
|||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.StringTokenizer;
|
||||
import java.util.TreeSet;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.math.NumberUtils;
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.completer.StringsCompleter;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import jline.console.completer.ArgumentCompleter.ArgumentList;
|
||||
import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter;
|
||||
|
||||
import static org.apache.commons.lang.StringUtils.isBlank;
|
||||
|
||||
/**
|
||||
* SQL auto complete functionality for the JdbcInterpreter.
|
||||
*/
|
||||
public class SqlCompleter extends StringsCompleter {
|
||||
public class SqlCompleter {
|
||||
|
||||
private static Logger logger = LoggerFactory.getLogger(SqlCompleter.class);
|
||||
|
||||
|
|
@ -67,8 +77,7 @@ public class SqlCompleter extends StringsCompleter {
|
|||
*/
|
||||
private StringsCompleter keywordCompleter = new StringsCompleter();
|
||||
|
||||
@Override
|
||||
public int complete(String buffer, int cursor, List<CharSequence> candidates) {
|
||||
public int complete(String buffer, int cursor, List<InterpreterCompletion> candidates) {
|
||||
|
||||
logger.debug("Complete with buffer = " + buffer + ", cursor = " + cursor);
|
||||
|
||||
|
|
@ -76,21 +85,36 @@ public class SqlCompleter extends StringsCompleter {
|
|||
// white spaces.
|
||||
ArgumentList argumentList = sqlDelimiter.delimit(buffer, cursor);
|
||||
|
||||
String beforeCursorBuffer = buffer.substring(0,
|
||||
Math.min(cursor, buffer.length())).toUpperCase();
|
||||
Pattern whitespaceEndPatter = Pattern.compile("\\s$");
|
||||
String cursorArgument = null;
|
||||
int argumentPosition;
|
||||
if (buffer.length() == 0 || whitespaceEndPatter.matcher(buffer).find()) {
|
||||
argumentPosition = buffer.length() - 1;
|
||||
} else {
|
||||
cursorArgument = argumentList.getCursorArgument();
|
||||
argumentPosition = argumentList.getArgumentPosition();
|
||||
}
|
||||
|
||||
// check what sql is and where cursor is to allow column completion or not
|
||||
boolean isColumnAllowed = true;
|
||||
if (beforeCursorBuffer.contains("SELECT ") && beforeCursorBuffer.contains(" FROM ")
|
||||
&& !beforeCursorBuffer.contains(" WHERE "))
|
||||
isColumnAllowed = false;
|
||||
if (buffer.length() > 0) {
|
||||
String beforeCursorBuffer = buffer.substring(0,
|
||||
Math.min(cursor, buffer.length())).toUpperCase();
|
||||
// check what sql is and where cursor is to allow column completion or not
|
||||
if (beforeCursorBuffer.contains("SELECT ") && beforeCursorBuffer.contains(" FROM ")
|
||||
&& !beforeCursorBuffer.contains(" WHERE "))
|
||||
isColumnAllowed = false;
|
||||
}
|
||||
|
||||
int complete = completeName(argumentList.getCursorArgument(),
|
||||
argumentList.getArgumentPosition(), candidates,
|
||||
int complete = completeName(cursorArgument, argumentPosition, candidates,
|
||||
findAliasesInSQL(argumentList.getArguments()), isColumnAllowed);
|
||||
|
||||
if (candidates.size() == 1) {
|
||||
InterpreterCompletion interpreterCompletion = candidates.get(0);
|
||||
interpreterCompletion.setName(interpreterCompletion.getName() + " ");
|
||||
interpreterCompletion.setValue(interpreterCompletion.getValue() + " ");
|
||||
candidates.set(0, interpreterCompletion);
|
||||
}
|
||||
logger.debug("complete:" + complete + ", size:" + candidates.size());
|
||||
|
||||
return complete;
|
||||
}
|
||||
|
||||
|
|
@ -98,24 +122,26 @@ public class SqlCompleter extends StringsCompleter {
|
|||
* Return list of schema names within the database
|
||||
*
|
||||
* @param meta metadata from connection to database
|
||||
* @param schemaFilter a schema name pattern; must match the schema name
|
||||
* @param schemaFilters a schema name patterns; must match the schema name
|
||||
* as it is stored in the database; "" retrieves those without a schema;
|
||||
* <code>null</code> means that the schema name should not be used to narrow
|
||||
* the search; supports '%' and '_' symbols; for example "prod_v_%"
|
||||
* the search; supports '%'; for example "prod_v_%"
|
||||
* @return set of all schema names in the database
|
||||
*/
|
||||
private static Set<String> getSchemaNames(DatabaseMetaData meta, String schemaFilter) {
|
||||
private static Set<String> getSchemaNames(DatabaseMetaData meta, List<String> schemaFilters) {
|
||||
Set<String> res = new HashSet<>();
|
||||
try {
|
||||
ResultSet schemas = meta.getSchemas();
|
||||
try {
|
||||
while (schemas.next()) {
|
||||
String schemaName = schemas.getString("TABLE_SCHEM");
|
||||
if (schemaName == null)
|
||||
if (schemaName == null) {
|
||||
schemaName = "";
|
||||
if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
|
||||
schemaFilter.replace("_", ".").replace("%", ".*?"))) {
|
||||
res.add(schemaName);
|
||||
}
|
||||
for (String schemaFilter : schemaFilters) {
|
||||
if (schemaFilter.equals("") || schemaName.matches(schemaFilter.replace("%", ".*?"))) {
|
||||
res.add(schemaName);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
|
|
@ -131,22 +157,23 @@ public class SqlCompleter extends StringsCompleter {
|
|||
* Return list of catalog names within the database
|
||||
*
|
||||
* @param meta metadata from connection to database
|
||||
* @param schemaFilter a catalog name pattern; must match the catalog name
|
||||
* @param schemaFilters a catalog name patterns; must match the catalog name
|
||||
* as it is stored in the database; "" retrieves those without a catalog;
|
||||
* <code>null</code> means that the schema name should not be used to narrow
|
||||
* the search; supports '%' and '_' symbols; for example "prod_v_%"
|
||||
* the search; supports '%'; for example "prod_v_%"
|
||||
* @return set of all catalog names in the database
|
||||
*/
|
||||
private static Set<String> getCatalogNames(DatabaseMetaData meta, String schemaFilter) {
|
||||
private static Set<String> getCatalogNames(DatabaseMetaData meta, List<String> schemaFilters) {
|
||||
Set<String> res = new HashSet<>();
|
||||
try {
|
||||
ResultSet schemas = meta.getCatalogs();
|
||||
try {
|
||||
while (schemas.next()) {
|
||||
String schemaName = schemas.getString("TABLE_CAT");
|
||||
if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
|
||||
schemaFilter.replace("_", ".").replace("%", ".*?"))) {
|
||||
res.add(schemaName);
|
||||
for (String schemaFilter : schemaFilters) {
|
||||
if (schemaFilter.equals("") || schemaName.matches(schemaFilter.replace("%", ".*?"))) {
|
||||
res.add(schemaName);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
|
|
@ -166,7 +193,7 @@ public class SqlCompleter extends StringsCompleter {
|
|||
* @param schemaFilter a schema name pattern; must match the schema name
|
||||
* as it is stored in the database; "" retrieves those without a schema;
|
||||
* <code>null</code> means that the schema name should not be used to narrow
|
||||
* the search; supports '%' and '_' symbols; for example "prod_v_%"
|
||||
* the search; supports '%'; for example "prod_v_%"
|
||||
* @param tables function fills this map, for every schema name adds
|
||||
* set of table names within the schema
|
||||
* @param columns function fills this map, for every table name adds set
|
||||
|
|
@ -177,19 +204,27 @@ public class SqlCompleter extends StringsCompleter {
|
|||
Map<String, Set<String>> tables,
|
||||
Map<String, Set<String>> columns) {
|
||||
try {
|
||||
ResultSet cols = meta.getColumns(catalogName, schemaFilter, "%",
|
||||
"%");
|
||||
ResultSet cols = meta.getColumns(catalogName, StringUtils.EMPTY, "%", "%");
|
||||
try {
|
||||
while (cols.next()) {
|
||||
String schema = cols.getString("TABLE_SCHEM");
|
||||
if (schema == null) schema = cols.getString("TABLE_CAT");
|
||||
if (schema == null) {
|
||||
schema = cols.getString("TABLE_CAT");
|
||||
}
|
||||
if (!schemaFilter.equals("") && !schema.matches(schemaFilter.replace("%", ".*?"))) {
|
||||
continue;
|
||||
}
|
||||
String table = cols.getString("TABLE_NAME");
|
||||
String column = cols.getString("COLUMN_NAME");
|
||||
if (!isBlank(table)) {
|
||||
String schemaTable = schema + "." + table;
|
||||
if (!columns.containsKey(schemaTable)) columns.put(schemaTable, new HashSet<String>());
|
||||
if (!columns.containsKey(schemaTable)) {
|
||||
columns.put(schemaTable, new HashSet<String>());
|
||||
}
|
||||
columns.get(schemaTable).add(column);
|
||||
if (!tables.containsKey(schema)) tables.put(schema, new HashSet<String>());
|
||||
if (!tables.containsKey(schema)) {
|
||||
tables.put(schema, new HashSet<String>());
|
||||
}
|
||||
tables.get(schema).add(table);
|
||||
}
|
||||
}
|
||||
|
|
@ -327,33 +362,31 @@ public class SqlCompleter extends StringsCompleter {
|
|||
* Initializes all local completers from database connection
|
||||
*
|
||||
* @param connection database connection
|
||||
* @param schemaFilter a schema name pattern; must match the schema name
|
||||
* as it is stored in the database; "" retrieves those without a schema;
|
||||
* <code>null</code> means that the schema name should not be used to narrow
|
||||
* the search; supports '%' and '_' symbols; for example "prod_v_%"
|
||||
* @param schemaFiltersString a comma separated schema name patterns; supports '%' symbol;
|
||||
* for example "prod_v_%,prod_t_%"
|
||||
*/
|
||||
public void initFromConnection(Connection connection, String schemaFilter) {
|
||||
public void initFromConnection(Connection connection, String schemaFiltersString) {
|
||||
if (schemaFiltersString == null) {
|
||||
schemaFiltersString = StringUtils.EMPTY;
|
||||
}
|
||||
List<String> schemaFilters = Arrays.asList(schemaFiltersString.split(","));
|
||||
|
||||
try {
|
||||
try (Connection c = connection) {
|
||||
Map<String, Set<String>> tables = new HashMap<>();
|
||||
Map<String, Set<String>> columns = new HashMap<>();
|
||||
Set<String> schemas = new HashSet<>();
|
||||
Set<String> catalogs = new HashSet<>();
|
||||
Set<String> keywords = getSqlKeywordsCompletions(connection);
|
||||
if (connection != null) {
|
||||
schemas = getSchemaNames(connection.getMetaData(), schemaFilter);
|
||||
catalogs = getCatalogNames(connection.getMetaData(), schemaFilter);
|
||||
|
||||
if (!"".equals(connection.getCatalog())) {
|
||||
if (schemas.size() == 0 )
|
||||
schemas.add(connection.getCatalog());
|
||||
fillTableAndColumnNames(connection.getCatalog(), connection.getMetaData(), schemaFilter,
|
||||
tables, columns);
|
||||
} else {
|
||||
if (schemas.size() == 0) schemas.addAll(catalogs);
|
||||
for (String catalog : catalogs) {
|
||||
fillTableAndColumnNames(catalog, connection.getMetaData(), schemaFilter, tables,
|
||||
columns);
|
||||
schemas = getSchemaNames(connection.getMetaData(), schemaFilters);
|
||||
catalogs = getCatalogNames(connection.getMetaData(), schemaFilters);
|
||||
if (schemas.size() == 0) {
|
||||
schemas.addAll(catalogs);
|
||||
}
|
||||
for (String schema : schemas) {
|
||||
for (String schemaFilter : schemaFilters) {
|
||||
fillTableAndColumnNames(schema, connection.getMetaData(), schemaFilter, tables,
|
||||
columns);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -408,8 +441,18 @@ public class SqlCompleter extends StringsCompleter {
|
|||
*/
|
||||
private int completeTable(String schema, String buffer, int cursor,
|
||||
List<CharSequence> candidates) {
|
||||
if (schema == null) {
|
||||
int res = -1;
|
||||
Set<CharSequence> candidatesSet = new HashSet<>();
|
||||
for (StringsCompleter stringsCompleter : tablesCompleters.values()) {
|
||||
int resTable = stringsCompleter.complete(buffer, cursor, candidatesSet);
|
||||
res = Math.max(res, resTable);
|
||||
}
|
||||
candidates.addAll(candidatesSet);
|
||||
return res;
|
||||
}
|
||||
// Wrong schema
|
||||
if (!tablesCompleters.containsKey(schema))
|
||||
if (!tablesCompleters.containsKey(schema) && schema != null)
|
||||
return -1;
|
||||
else
|
||||
return tablesCompleters.get(schema).complete(buffer, cursor, candidates);
|
||||
|
|
@ -422,12 +465,23 @@ public class SqlCompleter extends StringsCompleter {
|
|||
*/
|
||||
private int completeColumn(String schema, String table, String buffer, int cursor,
|
||||
List<CharSequence> candidates) {
|
||||
if (table == null && schema == null) {
|
||||
int res = -1;
|
||||
Set<CharSequence> candidatesSet = new HashSet<>();
|
||||
for (StringsCompleter stringsCompleter : columnsCompleters.values()) {
|
||||
int resColumn = stringsCompleter.complete(buffer, cursor, candidatesSet);
|
||||
res = Math.max(res, resColumn);
|
||||
}
|
||||
candidates.addAll(candidatesSet);
|
||||
return res;
|
||||
}
|
||||
// Wrong schema or wrong table
|
||||
if (!tablesCompleters.containsKey(schema) ||
|
||||
!columnsCompleters.containsKey(schema + "." + table))
|
||||
!columnsCompleters.containsKey(schema + "." + table)) {
|
||||
return -1;
|
||||
else
|
||||
} else {
|
||||
return columnsCompleters.get(schema + "." + table).complete(buffer, cursor, candidates);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -438,32 +492,43 @@ public class SqlCompleter extends StringsCompleter {
|
|||
* @param isColumnAllowed if false the function will not search and complete columns
|
||||
* @return -1 in case of no candidates found, 0 otherwise
|
||||
*/
|
||||
public int completeName(String buffer, int cursor, List<CharSequence> candidates,
|
||||
public int completeName(String buffer, int cursor, List<InterpreterCompletion> candidates,
|
||||
Map<String, String> aliases, boolean isColumnAllowed) {
|
||||
|
||||
if (buffer == null) buffer = "";
|
||||
|
||||
// no need to process after first point after cursor
|
||||
int nextPointPos = buffer.indexOf('.', cursor);
|
||||
if (nextPointPos != -1) buffer = buffer.substring(0, nextPointPos);
|
||||
|
||||
// points divide the name to the schema, table and column - find them
|
||||
int pointPos1 = buffer.indexOf('.');
|
||||
int pointPos2 = buffer.indexOf('.', pointPos1 + 1);
|
||||
int pointPos1 = -1;
|
||||
int pointPos2 = -1;
|
||||
|
||||
if (StringUtils.isNotEmpty(buffer)) {
|
||||
if (buffer.length() > cursor) {
|
||||
buffer = buffer.substring(0, cursor + 1);
|
||||
}
|
||||
pointPos1 = buffer.indexOf('.');
|
||||
pointPos2 = buffer.indexOf('.', pointPos1 + 1);
|
||||
}
|
||||
// find schema and table name if they are
|
||||
String schema;
|
||||
String table;
|
||||
String column;
|
||||
if (pointPos1 == -1) { // process only schema or keyword case
|
||||
schema = buffer;
|
||||
int keywordsRes = completeKeyword(buffer, cursor, candidates);
|
||||
|
||||
if (pointPos1 == -1) { // process all
|
||||
List<CharSequence> keywordsCandidates = new ArrayList();
|
||||
List<CharSequence> schemaCandidates = new ArrayList<>();
|
||||
int schemaRes = completeSchema(schema, cursor, schemaCandidates);
|
||||
candidates.addAll(schemaCandidates);
|
||||
return Math.max(keywordsRes, schemaRes);
|
||||
}
|
||||
else {
|
||||
List<CharSequence> tableCandidates = new ArrayList<>();
|
||||
List<CharSequence> columnCandidates = new ArrayList<>();
|
||||
int keywordsRes = completeKeyword(buffer, cursor, keywordsCandidates);
|
||||
int schemaRes = completeSchema(buffer, cursor, schemaCandidates);
|
||||
int tableRes = completeTable(null, buffer, cursor, tableCandidates);
|
||||
int columnRes = -1;
|
||||
if (isColumnAllowed) {
|
||||
columnRes = completeColumn(null, null, buffer, cursor, columnCandidates);
|
||||
}
|
||||
addCompletions(candidates, keywordsCandidates, CompletionType.keyword.name());
|
||||
addCompletions(candidates, schemaCandidates, CompletionType.schema.name());
|
||||
addCompletions(candidates, tableCandidates, CompletionType.table.name());
|
||||
addCompletions(candidates, columnCandidates, CompletionType.column.name());
|
||||
return NumberUtils.max(new int[]{keywordsRes, schemaRes, tableRes, columnRes});
|
||||
} else {
|
||||
schema = buffer.substring(0, pointPos1);
|
||||
if (aliases.containsKey(schema)) { // process alias case
|
||||
String alias = aliases.get(schema);
|
||||
|
|
@ -471,26 +536,40 @@ public class SqlCompleter extends StringsCompleter {
|
|||
schema = alias.substring(0, pointPos);
|
||||
table = alias.substring(pointPos + 1);
|
||||
column = buffer.substring(pointPos1 + 1);
|
||||
}
|
||||
else if (pointPos2 == -1) { // process schema.table case
|
||||
} else if (pointPos2 == -1) { // process schema.table case
|
||||
List<CharSequence> tableCandidates = new ArrayList();
|
||||
table = buffer.substring(pointPos1 + 1);
|
||||
return completeTable(schema, table, cursor - pointPos1 - 1, candidates);
|
||||
}
|
||||
else {
|
||||
int tableRes = completeTable(schema, table, cursor - pointPos1 - 1, tableCandidates);
|
||||
addCompletions(candidates, tableCandidates, CompletionType.table.name());
|
||||
return tableRes;
|
||||
} else {
|
||||
table = buffer.substring(pointPos1 + 1, pointPos2);
|
||||
column = buffer.substring(pointPos2 + 1);
|
||||
}
|
||||
}
|
||||
|
||||
// here in case of column
|
||||
if (isColumnAllowed)
|
||||
return completeColumn(schema, table, column, cursor - pointPos2 - 1, candidates);
|
||||
else
|
||||
return -1;
|
||||
if (table != null && isColumnAllowed) {
|
||||
List<CharSequence> columnCandidates = new ArrayList();
|
||||
int columnRes = completeColumn(schema, table, column, cursor - pointPos2 - 1,
|
||||
columnCandidates);
|
||||
addCompletions(candidates, columnCandidates, CompletionType.column.name());
|
||||
return columnRes;
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
// test purpose only
|
||||
WhitespaceArgumentDelimiter getSqlDelimiter() {
|
||||
return this.sqlDelimiter;
|
||||
}
|
||||
|
||||
private void addCompletions(List<InterpreterCompletion> interpreterCompletions,
|
||||
List<CharSequence> candidates, String meta) {
|
||||
for (CharSequence candidate : candidates) {
|
||||
interpreterCompletions.add(new InterpreterCompletion(candidate.toString(),
|
||||
candidate.toString(), meta));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
ABSOLUTE,ACTION,ADD,ALL,ALLOCATE,ALTER,AND,ANY,ARE,AS,ASC,ASSERTION,AT,AUTHORIZATION,AVG,BEGIN,BETWEEN,BIT,BIT_LENGTH,BOTH,BY,CASCADE,CASCADED,CASE,CAST,CATALOG,CHAR,CHARACTER,CHAR_LENGTH,CHARACTER_LENGTH,CHECK,CLOSE,CLUSTER,COALESCE,COLLATE,COLLATION,COLUMN,COMMIT,CONNECT,CONNECTION,CONSTRAINT,CONSTRAINTS,CONTINUE,CONVERT,CORRESPONDING,COUNT,CREATE,CROSS,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATE,DAY,DEALLOCATE,DEC,DECIMAL,DECLARE,DEFAULT,DEFERRABLE,DEFERRED,DELETE,DESC,DESCRIBE,DESCRIPTOR,DIAGNOSTICS,DISCONNECT,DISTINCT,DOMAIN,DOUBLE,DROP,ELSE,END,END-EXEC,ESCAPE,EXCEPT,EXCEPTION,EXEC,EXECUTE,EXISTS,EXTERNAL,EXTRACT,FALSE,FETCH,FIRST,FLOAT,FOR,FOREIGN,FOUND,FROM,FULL,GET,GLOBAL,GO,GOTO,GRANT,GROUP,HAVING,HOUR,IDENTITY,IMMEDIATE,IN,INDICATOR,INITIALLY,INNER,INPUT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,IS,ISOLATION,JOIN,KEY,LANGUAGE,LAST,LEADING,LEFT,LEVEL,LIKE,LOCAL,LOWER,MATCH,MAX,MIN,MINUTE,MODULE,MONTH,NAMES,NATIONAL,NATURAL,NCHAR,NEXT,NO,NOT,NULL,NULLIF,NUMERIC,OCTET_LENGTH,OF,ON,ONLY,OPEN,OPTION,OR,ORDER,OUTER,OUTPUT,OVERLAPS,OVERWRITE,PAD,PARTIAL,PARTITION,POSITION,PRECISION,PREPARE,PRESERVE,PRIMARY,PRIOR,PRIVILEGES,PROCEDURE,PUBLIC,READ,REAL,REFERENCES,RELATIVE,RESTRICT,REVOKE,RIGHT,ROLLBACK,ROWS,SCHEMA,SCROLL,SECOND,SECTION,SELECT,SESSION,SESSION_USER,SET,SIZE,SMALLINT,SOME,SPACE,SQL,SQLCODE,SQLERROR,SQLSTATE,SUBSTRING,SUM,SYSTEM_USER,TABLE,TEMPORARY,THEN,TIME,TIMESTAMP,TIMEZONE_HOUR,TIMEZONE_MINUTE,TO,TRAILING,TRANSACTION,TRANSLATE,TRANSLATION,TRIM,TRUE,UNION,UNIQUE,UNKNOWN,UPDATE,UPPER,USAGE,USER,USING,VALUE,VALUES,VARCHAR,VARYING,VIEW,WHEN,WHENEVER,WHERE,WITH,WORK,WRITE,YEAR,ZONE,ADA,C,CATALOG_NAME,CHARACTER_SET_CATALOG,CHARACTER_SET_NAME,CHARACTER_SET_SCHEMA,CLASS_ORIGIN,COBOL,COLLATION_CATALOG,COLLATION_NAME,COLLATION_SCHEMA,COLUMN_NAME,COMMAND_FUNCTION,COMMITTED,CONDITION_NUMBER,CONNECTION_NAME,CONSTRAINT_CATALOG,CONSTRAINT_NAME,CONSTRAINT_SCHEMA,CURSOR_NAME,DATA,DATETIME_INTERVAL_CODE,DATETIME_INTERVAL_PRECISION,DYNAMIC_FUNCTION,FORTRAN,LENGTH,MESSAGE_LENGTH,MESSAGE_OCTET_LENGTH,MESSAGE_TEXT,MORE,MUMPS,NAME,NULLABLE,NUMBER,PASCAL,PLI,REPEATABLE,RETURNED_LENGTH,RETURNED_OCTET_LENGTH,RETURNED_SQLSTATE,ROW_COUNT,SCALE,SCHEMA_NAME,SERIALIZABLE,SERVER_NAME,SUBCLASS_ORIGIN,TABLE_NAME,TYPE,UNCOMMITTED,UNNAMED,LIMIT
|
||||
absolute,action,add,all,allocate,alter,and,any,are,as,asc,assertion,at,authorization,avg,begin,between,bit,bit_length,both,by,cascade,cascaded,case,cast,catalog,char,character,char_length,character_length,check,close,cluster,coalesce,collate,collation,column,commit,connect,connection,constraint,constraints,continue,convert,corresponding,count,create,cross,current,current_date,current_time,current_timestamp,current_user,cursor,date,day,deallocate,dec,decimal,declare,default,deferrable,deferred,delete,desc,describe,descriptor,diagnostics,disconnect,distinct,domain,double,drop,else,end,end-exec,escape,except,exception,exec,execute,exists,external,extract,false,fetch,first,float,for,foreign,found,from,full,get,global,go,goto,grant,group,having,hour,identity,immediate,in,indicator,initially,inner,input,insensitive,insert,int,integer,intersect,interval,into,is,isolation,join,key,language,last,leading,left,level,like,local,lower,match,max,min,minute,module,month,names,national,natural,nchar,next,no,not,null,nullif,numeric,octet_length,of,on,only,open,option,or,order,outer,output,overlaps,overwrite,pad,partial,partition,position,precision,prepare,preserve,primary,prior,privileges,procedure,public,read,real,references,relative,restrict,revoke,right,rollback,rows,schema,scroll,second,section,select,session,session_user,set,size,smallint,some,space,sql,sqlcode,sqlerror,sqlstate,substring,sum,system_user,table,temporary,then,time,timestamp,timezone_hour,timezone_minute,to,trailing,transaction,translate,translation,trim,true,union,unique,unknown,update,upper,usage,user,using,value,values,varchar,varying,view,when,whenever,where,with,work,write,year,zone,ada,c,catalog_name,character_set_catalog,character_set_name,character_set_schema,class_origin,cobol,collation_catalog,collation_name,collation_schema,column_name,command_function,committed,condition_number,connection_name,constraint_catalog,constraint_name,constraint_schema,cursor_name,data,datetime_interval_code,datetime_interval_precision,dynamic_function,fortran,length,message_length,message_octet_length,message_text,more,mumps,name,nullable,number,pascal,pli,repeatable,returned_length,returned_octet_length,returned_sqlstate,row_count,scale,schema_name,serializable,server_name,subclass_origin,table_name,type,uncommitted,unnamed,limit
|
||||
|
|
|
|||
|
|
@ -28,6 +28,12 @@
|
|||
"defaultValue": "org.postgresql.Driver",
|
||||
"description": "JDBC Driver Name"
|
||||
},
|
||||
"default.completer.schemaFilters": {
|
||||
"envName": null,
|
||||
"propertyName": "default.completer.schemaFilters",
|
||||
"defaultValue": "",
|
||||
"description": "Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)"
|
||||
},
|
||||
"default.precode": {
|
||||
"envName": null,
|
||||
"propertyName": "zeppelin.jdbc.precode",
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -32,6 +32,7 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterResult;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
|
|
@ -254,6 +255,8 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
|
||||
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
|
||||
assertEquals("ID\tNAME\na\ta_name\n", interpreterResult.message().get(0).getData());
|
||||
assertEquals(InterpreterResult.Type.HTML, interpreterResult.message().get(1).getType());
|
||||
assertTrue(interpreterResult.message().get(1).getData().contains("alert-warning"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -293,9 +296,9 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
|
||||
jdbcInterpreter.interpret("", interpreterContext);
|
||||
|
||||
List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 1);
|
||||
List<InterpreterCompletion> completionList = jdbcInterpreter.completion("sel", 3, null);
|
||||
|
||||
InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select ", "select ");
|
||||
InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select ", "select ", CompletionType.keyword.name());
|
||||
|
||||
assertEquals(1, completionList.size());
|
||||
assertEquals(true, completionList.contains(correctCompletionKeyword));
|
||||
|
|
@ -336,18 +339,18 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
* 'jdbc1' interpreter has user('dbuser')/password('dbpassword') property
|
||||
* 'jdbc2' interpreter doesn't have user/password property
|
||||
* 'user1' doesn't have Credential information.
|
||||
* 'user2' has 'jdbc2' Credential information that is same with database account.
|
||||
* 'user2' has 'jdbc2' Credential information that is 'user2Id' / 'user2Pw' as id and password
|
||||
*/
|
||||
|
||||
JDBCInterpreter jdbc1 = new JDBCInterpreter(getDBProperty("dbuser", "dbpassword"));
|
||||
JDBCInterpreter jdbc2 = new JDBCInterpreter(getDBProperty(null, null));
|
||||
JDBCInterpreter jdbc2 = new JDBCInterpreter(getDBProperty("", ""));
|
||||
|
||||
AuthenticationInfo user1Credential = getUserAuth("user1", null, null, null);
|
||||
AuthenticationInfo user2Credential = getUserAuth("user2", "jdbc.jdbc2", "dbuser", "dbpassword");
|
||||
AuthenticationInfo user2Credential = getUserAuth("user2", "jdbc.jdbc2", "user2Id","user2Pw");
|
||||
|
||||
// user1 runs jdbc1
|
||||
jdbc1.open();
|
||||
InterpreterContext ctx1 = new InterpreterContext("", "1", "jdbc.jdbc1", "", "", user1Credential,
|
||||
InterpreterContext ctx1 = new InterpreterContext("", "1", "jdbc1", "", "", user1Credential,
|
||||
null, null, null, null, null, null);
|
||||
jdbc1.interpret("", ctx1);
|
||||
|
||||
|
|
@ -358,7 +361,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
|
||||
// user1 runs jdbc2
|
||||
jdbc2.open();
|
||||
InterpreterContext ctx2 = new InterpreterContext("", "1", "jdbc.jdbc2", "", "", user1Credential,
|
||||
InterpreterContext ctx2 = new InterpreterContext("", "1", "jdbc2", "", "", user1Credential,
|
||||
null, null, null, null, null, null);
|
||||
jdbc2.interpret("", ctx2);
|
||||
|
||||
|
|
@ -369,7 +372,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
|
||||
// user2 runs jdbc1
|
||||
jdbc1.open();
|
||||
InterpreterContext ctx3 = new InterpreterContext("", "1", "jdbc.jdbc1", "", "", user2Credential,
|
||||
InterpreterContext ctx3 = new InterpreterContext("", "1", "jdbc1", "", "", user2Credential,
|
||||
null, null, null, null, null, null);
|
||||
jdbc1.interpret("", ctx3);
|
||||
|
||||
|
|
@ -380,13 +383,13 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
|
||||
// user2 runs jdbc2
|
||||
jdbc2.open();
|
||||
InterpreterContext ctx4 = new InterpreterContext("", "1", "jdbc.jdbc2", "", "", user2Credential,
|
||||
InterpreterContext ctx4 = new InterpreterContext("", "1", "jdbc2", "", "", user2Credential,
|
||||
null, null, null, null, null, null);
|
||||
jdbc2.interpret("", ctx4);
|
||||
|
||||
JDBCUserConfigurations user2JDBC2Conf = jdbc2.getJDBCConfiguration("user2");
|
||||
assertNull(user2JDBC2Conf.getPropertyMap("default").get("user"));
|
||||
assertNull(user2JDBC2Conf.getPropertyMap("default").get("password"));
|
||||
assertEquals("user2Id", user2JDBC2Conf.getPropertyMap("default").get("user"));
|
||||
assertEquals("user2Pw", user2JDBC2Conf.getPropertyMap("default").get("password"));
|
||||
jdbc2.close();
|
||||
}
|
||||
|
||||
|
|
@ -452,4 +455,32 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
|
||||
assertEquals("@TESTVARIABLE\n2\n", interpreterResult.message().get(0).getData());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExcludingComments() throws SQLException, IOException {
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty("common.max_count", "1000");
|
||||
properties.setProperty("common.max_retry", "3");
|
||||
properties.setProperty("default.driver", "org.h2.Driver");
|
||||
properties.setProperty("default.url", getJdbcConnection());
|
||||
properties.setProperty("default.user", "");
|
||||
properties.setProperty("default.password", "");
|
||||
JDBCInterpreter t = new JDBCInterpreter(properties);
|
||||
t.open();
|
||||
|
||||
String sqlQuery = "/* ; */\n" +
|
||||
"-- /* comment\n" +
|
||||
"--select * from test_table\n" +
|
||||
"select * from test_table; /* some comment ; */\n" +
|
||||
"/*\n" +
|
||||
"select * from test_table;\n" +
|
||||
"*/\n" +
|
||||
"-- a ; b\n" +
|
||||
"select * from test_table WHERE ID = ';--';\n" +
|
||||
"select * from test_table WHERE ID = '/*' -- test";
|
||||
|
||||
InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
|
||||
assertEquals(3, interpreterResult.message().size());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,18 +14,26 @@
|
|||
*/
|
||||
package org.apache.zeppelin.jdbc;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
|
||||
import jline.console.completer.ArgumentCompleter;
|
||||
import jline.console.completer.Completer;
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.zeppelin.completer.CompletionType;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.*;
|
||||
import com.google.common.base.Joiner;
|
||||
import jline.console.completer.ArgumentCompleter;
|
||||
|
||||
import static com.google.common.collect.Sets.newHashSet;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
|
@ -34,18 +42,18 @@ import static org.junit.Assert.assertTrue;
|
|||
/**
|
||||
* SQL completer unit tests
|
||||
*/
|
||||
public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
||||
public class SqlCompleterTest {
|
||||
|
||||
public class CompleterTester {
|
||||
|
||||
private Completer completer;
|
||||
private SqlCompleter completer;
|
||||
|
||||
private String buffer;
|
||||
private int fromCursor;
|
||||
private int toCursor;
|
||||
private Set<String> expectedCompletions;
|
||||
private Set<InterpreterCompletion> expectedCompletions;
|
||||
|
||||
public CompleterTester(Completer completer) {
|
||||
public CompleterTester(SqlCompleter completer) {
|
||||
this.completer = completer;
|
||||
}
|
||||
|
||||
|
|
@ -64,7 +72,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
return this;
|
||||
}
|
||||
|
||||
public CompleterTester expect(Set<String> expectedCompletions) {
|
||||
public CompleterTester expect(Set<InterpreterCompletion> expectedCompletions) {
|
||||
this.expectedCompletions = expectedCompletions;
|
||||
return this;
|
||||
}
|
||||
|
|
@ -75,9 +83,13 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
private void expectedCompletions(String buffer, int cursor, Set<String> expected) {
|
||||
private void expectedCompletions(String buffer, int cursor,
|
||||
Set<InterpreterCompletion> expected) {
|
||||
if (StringUtils.isNotEmpty(buffer) && buffer.length() > cursor) {
|
||||
buffer = buffer.substring(0, cursor + 1);
|
||||
}
|
||||
|
||||
ArrayList<CharSequence> candidates = new ArrayList<>();
|
||||
List<InterpreterCompletion> candidates = new ArrayList<>();
|
||||
|
||||
completer.complete(buffer, cursor, candidates);
|
||||
|
||||
|
|
@ -85,11 +97,15 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
|
||||
logger.info(explain);
|
||||
|
||||
assertEquals("Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] "
|
||||
+ explain, expected, newHashSet(candidates));
|
||||
Assert.assertEquals("Buffer [" + buffer.replace(" ", ".") + "] and Cursor[" + cursor + "] "
|
||||
+ explain, expected, newHashSet(candidates));
|
||||
}
|
||||
|
||||
private String explain(String buffer, int cursor, ArrayList<CharSequence> candidates) {
|
||||
private String explain(String buffer, int cursor, List<InterpreterCompletion> candidates) {
|
||||
List<String> cndidateStrings = new ArrayList<>();
|
||||
for (InterpreterCompletion candidate : candidates) {
|
||||
cndidateStrings.add(candidate.getValue());
|
||||
}
|
||||
StringBuffer sb = new StringBuffer();
|
||||
|
||||
for (int i = 0; i <= Math.max(cursor, buffer.length()); i++) {
|
||||
|
|
@ -109,7 +125,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
sb.append(")");
|
||||
}
|
||||
}
|
||||
sb.append(" >> [").append(Joiner.on(",").join(candidates)).append("]");
|
||||
sb.append(" >> [").append(Joiner.on(",").join(cndidateStrings)).append("]");
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
|
@ -122,7 +138,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
private CompleterTester tester;
|
||||
|
||||
private ArgumentCompleter.WhitespaceArgumentDelimiter delimiter =
|
||||
new ArgumentCompleter.WhitespaceArgumentDelimiter();
|
||||
new ArgumentCompleter.WhitespaceArgumentDelimiter();
|
||||
|
||||
private SqlCompleter sqlCompleter = new SqlCompleter();
|
||||
|
||||
|
|
@ -178,7 +194,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testFindAliasesInSQL_Simple(){
|
||||
public void testFindAliasesInSQL_Simple() {
|
||||
String sql = "select * from prod_emart.financial_account a";
|
||||
Map<String, String> res = sqlCompleter.findAliasesInSQL(delimiter.delimit(sql, 0).getArguments());
|
||||
assertEquals(1, res.size());
|
||||
|
|
@ -186,7 +202,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testFindAliasesInSQL_Two(){
|
||||
public void testFindAliasesInSQL_Two() {
|
||||
String sql = "select * from prod_dds.financial_account a, prod_dds.customer b";
|
||||
Map<String, String> res = sqlCompleter.findAliasesInSQL(sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments());
|
||||
assertEquals(2, res.size());
|
||||
|
|
@ -195,7 +211,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testFindAliasesInSQL_WrongTables(){
|
||||
public void testFindAliasesInSQL_WrongTables() {
|
||||
String sql = "select * from prod_ddsxx.financial_account a, prod_dds.customerxx b";
|
||||
Map<String, String> res = sqlCompleter.findAliasesInSQL(sqlCompleter.getSqlDelimiter().delimit(sql, 0).getArguments());
|
||||
assertEquals(0, res.size());
|
||||
|
|
@ -205,116 +221,145 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
|
|||
public void testCompleteName_Empty() {
|
||||
String buffer = "";
|
||||
int cursor = 0;
|
||||
List<CharSequence> candidates = new ArrayList<>();
|
||||
List<InterpreterCompletion> candidates = new ArrayList<>();
|
||||
Map<String, String> aliases = new HashMap<>();
|
||||
sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
|
||||
assertEquals(9, candidates.size());
|
||||
assertTrue(candidates.contains("prod_dds"));
|
||||
assertTrue(candidates.contains("prod_emart"));
|
||||
assertTrue(candidates.contains("SUM"));
|
||||
assertTrue(candidates.contains("SUBSTRING"));
|
||||
assertTrue(candidates.contains("SUBCLASS_ORIGIN"));
|
||||
assertTrue(candidates.contains("SELECT"));
|
||||
assertTrue(candidates.contains("ORDER"));
|
||||
assertTrue(candidates.contains("LIMIT"));
|
||||
assertTrue(candidates.contains("FROM"));
|
||||
sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
|
||||
assertEquals(17, candidates.size());
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("customer", "customer", CompletionType.table.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("name", "name", CompletionType.column.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompleteName_SimpleSchema() {
|
||||
String buffer = "prod_";
|
||||
int cursor = 3;
|
||||
List<CharSequence> candidates = new ArrayList<>();
|
||||
List<InterpreterCompletion> candidates = new ArrayList<>();
|
||||
Map<String, String> aliases = new HashMap<>();
|
||||
sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
|
||||
assertEquals(2, candidates.size());
|
||||
assertTrue(candidates.contains("prod_dds"));
|
||||
assertTrue(candidates.contains("prod_emart"));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompleteName_SimpleTable() {
|
||||
String buffer = "prod_dds.fin";
|
||||
int cursor = 11;
|
||||
List<CharSequence> candidates = new ArrayList<>();
|
||||
List<InterpreterCompletion> candidates = new ArrayList<>();
|
||||
Map<String, String> aliases = new HashMap<>();
|
||||
sqlCompleter.completeName(buffer, cursor, candidates, aliases, false);
|
||||
assertEquals(1, candidates.size());
|
||||
assertTrue(candidates.contains("financial_account "));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompleteName_SimpleColumn() {
|
||||
String buffer = "prod_dds.financial_account.acc";
|
||||
int cursor = 30;
|
||||
List<CharSequence> candidates = new ArrayList<>();
|
||||
List<InterpreterCompletion> candidates = new ArrayList<>();
|
||||
Map<String, String> aliases = new HashMap<>();
|
||||
sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
|
||||
assertEquals(2, candidates.size());
|
||||
assertTrue(candidates.contains("account_rk"));
|
||||
assertTrue(candidates.contains("account_id"));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompleteName_WithAlias() {
|
||||
String buffer = "a.acc";
|
||||
int cursor = 4;
|
||||
List<CharSequence> candidates = new ArrayList<>();
|
||||
List<InterpreterCompletion> candidates = new ArrayList<>();
|
||||
Map<String, String> aliases = new HashMap<>();
|
||||
aliases.put("a", "prod_dds.financial_account");
|
||||
sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
|
||||
assertEquals(2, candidates.size());
|
||||
assertTrue(candidates.contains("account_rk"));
|
||||
assertTrue(candidates.contains("account_id"));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompleteName_WithAliasAndPoint() {
|
||||
String buffer = "a.";
|
||||
int cursor = 2;
|
||||
List<CharSequence> candidates = new ArrayList<>();
|
||||
List<InterpreterCompletion> candidates = new ArrayList<>();
|
||||
Map<String, String> aliases = new HashMap<>();
|
||||
aliases.put("a", "prod_dds.financial_account");
|
||||
sqlCompleter.completeName(buffer, cursor, candidates, aliases, true);
|
||||
assertEquals(2, candidates.size());
|
||||
assertTrue(candidates.contains("account_rk"));
|
||||
assertTrue(candidates.contains("account_id"));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name())));
|
||||
assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name())));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSchemaAndTable() {
|
||||
String buffer = "select * from prod_v_emart.fi";
|
||||
tester.buffer(buffer).from(15).to(26).expect(newHashSet("prod_v_emart ")).test();
|
||||
tester.buffer(buffer).from(27).to(29).expect(newHashSet("financial_account ")).test();
|
||||
String buffer = "select * from prod_emart.fi";
|
||||
tester.buffer(buffer).from(19).to(23).expect(newHashSet(new InterpreterCompletion("prod_emart ", "prod_emart ", CompletionType.schema.name()))).test();
|
||||
tester.buffer(buffer).from(25).to(27).expect(newHashSet(new InterpreterCompletion("financial_account ", "financial_account ", CompletionType.table.name()))).test();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEdges() {
|
||||
String buffer = " ORDER ";
|
||||
tester.buffer(buffer).from(0).to(7).expect(newHashSet("ORDER ")).test();
|
||||
tester.buffer(buffer).from(8).to(15).expect(newHashSet("ORDER", "SUBCLASS_ORIGIN", "SUBSTRING",
|
||||
"prod_emart", "LIMIT", "SUM", "prod_dds", "SELECT", "FROM")).test();
|
||||
tester.buffer(buffer).from(2).to(6).expect(newHashSet(new InterpreterCompletion("ORDER ", "ORDER ", CompletionType.keyword.name()))).test();
|
||||
tester.buffer(buffer).from(0).to(1).expect(newHashSet(
|
||||
new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()),
|
||||
new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()),
|
||||
new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()),
|
||||
new InterpreterCompletion("customer", "customer", CompletionType.table.name()),
|
||||
new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()),
|
||||
new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()),
|
||||
new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name()),
|
||||
new InterpreterCompletion("name", "name", CompletionType.column.name()),
|
||||
new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name()),
|
||||
new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name())
|
||||
)).test();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultipleWords() {
|
||||
String buffer = "SELE FRO LIM";
|
||||
tester.buffer(buffer).from(0).to(4).expect(newHashSet("SELECT ")).test();
|
||||
tester.buffer(buffer).from(5).to(8).expect(newHashSet("FROM ")).test();
|
||||
tester.buffer(buffer).from(9).to(12).expect(newHashSet("LIMIT ")).test();
|
||||
tester.buffer(buffer).from(1).to(3).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test();
|
||||
tester.buffer(buffer).from(6).to(7).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test();
|
||||
tester.buffer(buffer).from(9).to(12).expect(newHashSet(new InterpreterCompletion("LIMIT ", "LIMIT ", CompletionType.keyword.name()))).test();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiLineBuffer() {
|
||||
String buffer = " \n SELE\nFRO";
|
||||
tester.buffer(buffer).from(0).to(7).expect(newHashSet("SELECT ")).test();
|
||||
tester.buffer(buffer).from(8).to(11).expect(newHashSet("FROM ")).test();
|
||||
tester.buffer(buffer).from(4).to(6).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test();
|
||||
tester.buffer(buffer).from(9).to(11).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultipleCompletionSuggestions() {
|
||||
String buffer = "SU";
|
||||
tester.buffer(buffer).from(0).to(2).expect(newHashSet("SUBCLASS_ORIGIN", "SUM", "SUBSTRING"))
|
||||
.test();
|
||||
tester.buffer(buffer).from(1).to(2).expect(newHashSet(
|
||||
new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()),
|
||||
new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name()))
|
||||
).test();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -95,7 +95,8 @@ public class KylinInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -420,7 +420,8 @@ public class LensInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import org.apache.http.client.HttpClient;
|
|||
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
|
||||
import org.apache.http.conn.ssl.SSLContexts;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.commons.lang.exception.ExceptionUtils;
|
||||
import org.apache.zeppelin.interpreter.*;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -42,13 +43,17 @@ import javax.net.ssl.SSLContext;
|
|||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.security.KeyStore;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Base class for livy interpreters.
|
||||
*/
|
||||
|
|
@ -67,9 +72,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
|
|||
protected LivyVersion livyVersion;
|
||||
private RestTemplate restTemplate;
|
||||
|
||||
// keep tracking the mapping between paragraphId and statementId, so that we can cancel the
|
||||
// statement after we execute it.
|
||||
private ConcurrentHashMap<String, Integer> paragraphId2StmtIdMap = new ConcurrentHashMap<>();
|
||||
Set<Object> paragraphsToCancel = Collections.newSetFromMap(
|
||||
new ConcurrentHashMap<Object, Boolean>());
|
||||
private ConcurrentHashMap<String, Integer> paragraphId2StmtProgressMap =
|
||||
new ConcurrentHashMap<>();
|
||||
|
||||
|
|
@ -162,21 +166,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
|
|||
|
||||
@Override
|
||||
public void cancel(InterpreterContext context) {
|
||||
if (livyVersion.isCancelSupported()) {
|
||||
String paraId = context.getParagraphId();
|
||||
Integer stmtId = paragraphId2StmtIdMap.get(paraId);
|
||||
try {
|
||||
if (stmtId != null) {
|
||||
cancelStatement(stmtId);
|
||||
}
|
||||
} catch (LivyException e) {
|
||||
LOGGER.error("Fail to cancel statement " + stmtId + " for paragraph " + paraId, e);
|
||||
} finally {
|
||||
paragraphId2StmtIdMap.remove(paraId);
|
||||
}
|
||||
} else {
|
||||
LOGGER.warn("cancel is not supported for this version of livy: " + livyVersion);
|
||||
}
|
||||
paragraphsToCancel.add(context.getParagraphId());
|
||||
LOGGER.info("Added paragraph " + context.getParagraphId() + " for cancellation.");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -260,11 +251,12 @@ public abstract class BaseLivyInterprereter extends Interpreter {
|
|||
}
|
||||
stmtInfo = executeStatement(new ExecuteRequest(code));
|
||||
}
|
||||
if (paragraphId != null) {
|
||||
paragraphId2StmtIdMap.put(paragraphId, stmtInfo.id);
|
||||
}
|
||||
// pull the statement status
|
||||
while (!stmtInfo.isAvailable()) {
|
||||
if (paragraphId != null && paragraphsToCancel.contains(paragraphId)) {
|
||||
cancel(stmtInfo.id, paragraphId);
|
||||
return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled");
|
||||
}
|
||||
try {
|
||||
Thread.sleep(pullStatusInterval);
|
||||
} catch (InterruptedException e) {
|
||||
|
|
@ -284,12 +276,29 @@ public abstract class BaseLivyInterprereter extends Interpreter {
|
|||
}
|
||||
} finally {
|
||||
if (paragraphId != null) {
|
||||
paragraphId2StmtIdMap.remove(paragraphId);
|
||||
paragraphId2StmtProgressMap.remove(paragraphId);
|
||||
paragraphsToCancel.remove(paragraphId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void cancel(int id, String paragraphId) {
|
||||
if (livyVersion.isCancelSupported()) {
|
||||
try {
|
||||
LOGGER.info("Cancelling statement " + id);
|
||||
cancelStatement(id);
|
||||
} catch (LivyException e) {
|
||||
LOGGER.error("Fail to cancel statement " + id + " for paragraph " + paragraphId, e);
|
||||
}
|
||||
finally {
|
||||
paragraphsToCancel.remove(paragraphId);
|
||||
}
|
||||
} else {
|
||||
LOGGER.warn("cancel is not supported for this version of livy: " + livyVersion);
|
||||
paragraphsToCancel.clear();
|
||||
}
|
||||
}
|
||||
|
||||
protected LivyVersion getLivyVersion() throws LivyException {
|
||||
return new LivyVersion((LivyVersionResponse.fromJson(callRestAPI("/version", "GET")).version));
|
||||
}
|
||||
|
|
@ -369,13 +378,11 @@ public abstract class BaseLivyInterprereter extends Interpreter {
|
|||
}
|
||||
|
||||
if (displayAppInfo) {
|
||||
//TODO(zjffdu), use multiple InterpreterResult to display appInfo
|
||||
InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
|
||||
interpreterResult.add(InterpreterResult.Type.TEXT, result);
|
||||
interpreterResult.add(result);
|
||||
String appInfoHtml = "<hr/>Spark Application Id: " + sessionInfo.appId + "<br/>"
|
||||
+ "Spark WebUI: <a href=\"" + sessionInfo.webUIAddress + "\">"
|
||||
+ sessionInfo.webUIAddress + "</a>";
|
||||
LOGGER.info("appInfoHtml:" + appInfoHtml);
|
||||
interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml);
|
||||
return interpreterResult;
|
||||
} else {
|
||||
|
|
@ -486,6 +493,8 @@ public abstract class BaseLivyInterprereter extends Interpreter {
|
|||
if (cause.getResponseBodyAsString().matches(SESSION_NOT_FOUND_PATTERN)) {
|
||||
throw new SessionNotFoundException(cause.getResponseBodyAsString());
|
||||
}
|
||||
throw new LivyException(cause.getResponseBodyAsString() + "\n"
|
||||
+ ExceptionUtils.getFullStackTrace(ExceptionUtils.getRootCause(e)));
|
||||
}
|
||||
throw new LivyException(e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -141,8 +141,8 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
|
|||
List<String> rows = parseSQLOutput(message.getData());
|
||||
result2.add(InterpreterResult.Type.TABLE, StringUtils.join(rows, "\n"));
|
||||
if (rows.size() >= (maxResult + 1)) {
|
||||
result2.add(InterpreterResult.Type.HTML,
|
||||
"<font color=red>Results are limited by " + maxResult + ".</font>");
|
||||
result2.add(ResultMessages.getExceedsLimitRowsMessage(maxResult,
|
||||
ZEPPELIN_LIVY_SPARK_SQL_MAX_RESULT));
|
||||
}
|
||||
} else {
|
||||
result2.add(message.getType(), message.getData());
|
||||
|
|
@ -229,6 +229,11 @@ public class LivySparkSQLInterpreter extends BaseLivyInterprereter {
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void cancel(InterpreterContext context) {
|
||||
sparkInterpreter.cancel(context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
this.sparkInterpreter.close();
|
||||
|
|
|
|||
|
|
@ -145,6 +145,13 @@ public class LivyInterpreterIT {
|
|||
assertTrue(result.message().get(0).getData().contains("defined object Person"));
|
||||
}
|
||||
|
||||
// html output
|
||||
String htmlCode = "println(\"%html <h1> hello </h1>\")";
|
||||
result = sparkInterpreter.interpret(htmlCode, context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertEquals(1, result.message().size());
|
||||
assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType());
|
||||
|
||||
// error
|
||||
result = sparkInterpreter.interpret("println(a)", context);
|
||||
assertEquals(InterpreterResult.Code.ERROR, result.code());
|
||||
|
|
@ -162,9 +169,9 @@ public class LivyInterpreterIT {
|
|||
Thread cancelThread = new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
// invoke cancel after 3 seconds to wait job starting
|
||||
// invoke cancel after 1 millisecond to wait job starting
|
||||
try {
|
||||
Thread.sleep(3000);
|
||||
Thread.sleep(1);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
|
@ -306,6 +313,88 @@ public class LivyInterpreterIT {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSparkSQLCancellation() {
|
||||
if (!checkPreCondition()) {
|
||||
return;
|
||||
}
|
||||
InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
|
||||
interpreterGroup.put("session_1", new ArrayList<Interpreter>());
|
||||
LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties);
|
||||
sparkInterpreter.setInterpreterGroup(interpreterGroup);
|
||||
interpreterGroup.get("session_1").add(sparkInterpreter);
|
||||
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
|
||||
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
|
||||
InterpreterOutput output = new InterpreterOutput(outputListener);
|
||||
final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
|
||||
"title", "text", authInfo, null, null, null, null, null, output);
|
||||
sparkInterpreter.open();
|
||||
|
||||
final LivySparkSQLInterpreter sqlInterpreter = new LivySparkSQLInterpreter(properties);
|
||||
interpreterGroup.get("session_1").add(sqlInterpreter);
|
||||
sqlInterpreter.setInterpreterGroup(interpreterGroup);
|
||||
sqlInterpreter.open();
|
||||
|
||||
try {
|
||||
// detect spark version
|
||||
InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertEquals(1, result.message().size());
|
||||
|
||||
boolean isSpark2 = isSpark2(sparkInterpreter, context);
|
||||
|
||||
// test DataFrame api
|
||||
if (!isSpark2) {
|
||||
result = sparkInterpreter.interpret(
|
||||
"val df=sqlContext.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
|
||||
+ "df.collect()", context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertEquals(1, result.message().size());
|
||||
assertTrue(result.message().get(0).getData()
|
||||
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
|
||||
} else {
|
||||
result = sparkInterpreter.interpret(
|
||||
"val df=spark.createDataFrame(Seq((\"hello\",20))).toDF(\"col_1\", \"col_2\")\n"
|
||||
+ "df.collect()", context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertEquals(1, result.message().size());
|
||||
assertTrue(result.message().get(0).getData()
|
||||
.contains("Array[org.apache.spark.sql.Row] = Array([hello,20])"));
|
||||
}
|
||||
sparkInterpreter.interpret("df.registerTempTable(\"df\")", context);
|
||||
|
||||
// cancel
|
||||
if (sqlInterpreter.getLivyVersion().newerThanEquals(LivyVersion.LIVY_0_3_0)) {
|
||||
Thread cancelThread = new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
sqlInterpreter.cancel(context);
|
||||
}
|
||||
};
|
||||
cancelThread.start();
|
||||
//sleep so that cancelThread performs a cancel.
|
||||
try {
|
||||
Thread.sleep(1);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
result = sqlInterpreter
|
||||
.interpret("select count(1) from df", context);
|
||||
if (result.code().equals(InterpreterResult.Code.ERROR)) {
|
||||
String message = result.message().get(0).getData();
|
||||
// 2 possibilities, sometimes livy doesn't return the real cancel exception
|
||||
assertTrue(message.contains("cancelled part of cancelled job group") ||
|
||||
message.contains("Job is cancelled"));
|
||||
}
|
||||
}
|
||||
} catch (LivyException e) {
|
||||
} finally {
|
||||
sparkInterpreter.close();
|
||||
sqlInterpreter.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStringWithTruncation() {
|
||||
if (!checkPreCondition()) {
|
||||
|
|
@ -495,9 +584,9 @@ public class LivyInterpreterIT {
|
|||
Thread cancelThread = new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
// invoke cancel after 3 seconds to wait job starting
|
||||
// invoke cancel after 1 millisecond to wait job starting
|
||||
try {
|
||||
Thread.sleep(3000);
|
||||
Thread.sleep(1);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
|
@ -544,8 +633,15 @@ public class LivyInterpreterIT {
|
|||
InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertEquals(2, result.message().size());
|
||||
|
||||
assertTrue(result.message().get(1).getData().contains("Spark Application Id"));
|
||||
|
||||
// html output
|
||||
String htmlCode = "println(\"%html <h1> hello </h1>\")";
|
||||
result = sparkInterpreter.interpret(htmlCode, context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertEquals(2, result.message().size());
|
||||
assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType());
|
||||
|
||||
} finally {
|
||||
sparkInterpreter.close();
|
||||
}
|
||||
|
|
@ -586,9 +682,9 @@ public class LivyInterpreterIT {
|
|||
Thread cancelThread = new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
// invoke cancel after 3 seconds to wait job starting
|
||||
// invoke cancel after 1 millisecond to wait job starting
|
||||
try {
|
||||
Thread.sleep(3000);
|
||||
Thread.sleep(1);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -124,7 +124,8 @@ public class Markdown extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@
|
|||
"msg": [
|
||||
{
|
||||
"type": "HTML",
|
||||
"data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eIn this tutorial we will go through some of the basic features of Zeppelin\u0026rsquo;s built-in matplotlib integration. \u003c/p\u003e\n\u003ch3\u003ePrerequisites\u003c/h3\u003e\n\u003cp\u003e\u003ccode\u003ematplotlib\u003c/code\u003e must be installed to your local python installation. (use \u003ccode\u003epip install matplotlib\u003c/code\u003e or \u003ccode\u003econda install matplotlib\u003c/code\u003e if you have \u003ccode\u003econda\u003c/code\u003e). Additionally, you will need Zeppelin\u0026rsquo;s matplotlib backend files which are usually found in \u003ccode\u003e$ZEPPELIN_HOME/lib/python\u003c/code\u003e. Although Zeppelin should automatically find this directory, it might be a good idea to add it to your \u003ccode\u003ePYTHONPATH\u003c/code\u003e just in case. \u003c/p\u003e\n\u003ch3\u003eInterpreters\u003c/h3\u003e\n\u003cp\u003eMost of the examples shown in this tutorial can be used interchangeably with either the \u003ccode\u003epython\u003c/code\u003e or \u003ccode\u003epyspark\u003c/code\u003e interpreters. Iterative plotting using the Angular Display System is currently only available for \u003ccode\u003epyspark\u003c/code\u003e, but this functionality will eventually be added to the base \u003ccode\u003epython\u003c/code\u003e interpreter. \u003c/p\u003e\n\u003ch3\u003emacOS\u003c/h3\u003e\n\u003cp\u003eMake sure locale is set, to avoid \u003ccode\u003eValueError: unknown locale: UTF-8\u003c/code\u003e\u003c/p\u003e\n\u003ch3\u003evirtualenv\u003c/h3\u003e\n\u003cp\u003eIn case you want to use virtualenv or conda env:\u003cbr/\u003e - configure python interpreter property -\u0026gt; \u003ccode\u003eabsolute/path/to/venv/bin/python\u003c/code\u003e\u003cbr/\u003e - see \u003cem\u003eWorking with Matplotlib in Virtual environments\u003c/em\u003e in the \u003ca href\u003d\"http://matplotlib.org/faq/virtualenv_faq.html\"\u003eMatplotlib FAQ\u003c/a\u003e\u003c/p\u003e\n\u003ch3\u003eA simple example\u003c/h3\u003e\n\u003cp\u003eLet\u0026rsquo;s start by making a very simple line plot:\u003c/p\u003e\n\u003c/div\u003e"
|
||||
"data": "\u003cdiv class\u003d\"markdown-body\"\u003e\n\u003ch2\u003eIntroduction\u003c/h2\u003e\n\u003cp\u003eIn this tutorial we will go through some of the basic features of Zeppelin\u0026rsquo;s built-in matplotlib integration. \u003c/p\u003e\n\u003ch3\u003ePrerequisites\u003c/h3\u003e\n\u003cp\u003e\u003ccode\u003ematplotlib\u003c/code\u003e must be installed to your local python installation. (use \u003ccode\u003epip install matplotlib\u003c/code\u003e or \u003ccode\u003econda install matplotlib\u003c/code\u003e if you have \u003ccode\u003econda\u003c/code\u003e). Additionally, you will need Zeppelin\u0026rsquo;s matplotlib backend files which are usually found in \u003ccode\u003e$ZEPPELIN_HOME/interpreter/lib/python\u003c/code\u003e. Although Zeppelin should automatically find this directory, it might be a good idea to add it to your \u003ccode\u003ePYTHONPATH\u003c/code\u003e just in case. \u003c/p\u003e\n\u003ch3\u003eInterpreters\u003c/h3\u003e\n\u003cp\u003eMost of the examples shown in this tutorial can be used interchangeably with either the \u003ccode\u003epython\u003c/code\u003e or \u003ccode\u003epyspark\u003c/code\u003e interpreters. Iterative plotting using the Angular Display System is currently only available for \u003ccode\u003epyspark\u003c/code\u003e, but this functionality will eventually be added to the base \u003ccode\u003epython\u003c/code\u003e interpreter. \u003c/p\u003e\n\u003ch3\u003emacOS\u003c/h3\u003e\n\u003cp\u003eMake sure locale is set, to avoid \u003ccode\u003eValueError: unknown locale: UTF-8\u003c/code\u003e\u003c/p\u003e\n\u003ch3\u003evirtualenv\u003c/h3\u003e\n\u003cp\u003eIn case you want to use virtualenv or conda env:\u003cbr/\u003e - configure python interpreter property -\u0026gt; \u003ccode\u003eabsolute/path/to/venv/bin/python\u003c/code\u003e\u003cbr/\u003e - see \u003cem\u003eWorking with Matplotlib in Virtual environments\u003c/em\u003e in the \u003ca href\u003d\"http://matplotlib.org/faq/virtualenv_faq.html\"\u003eMatplotlib FAQ\u003c/a\u003e\u003c/p\u003e\n\u003ch3\u003eA simple example\u003c/h3\u003e\n\u003cp\u003eLet\u0026rsquo;s start by making a very simple line plot:\u003c/p\u003e\n\u003c/div\u003e"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -47,6 +47,12 @@
|
|||
<artifactId>zeppelin-interpreter</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>jline</groupId>
|
||||
<artifactId>jline</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ import java.util.Properties;
|
|||
public class PigQueryInterpreter extends BasePigInterpreter {
|
||||
|
||||
private static Logger LOGGER = LoggerFactory.getLogger(PigQueryInterpreter.class);
|
||||
private static final String MAX_RESULTS = "zeppelin.pig.maxResult";
|
||||
private PigServer pigServer;
|
||||
private int maxResult;
|
||||
|
||||
|
|
@ -55,7 +56,7 @@ public class PigQueryInterpreter extends BasePigInterpreter {
|
|||
@Override
|
||||
public void open() {
|
||||
pigServer = getPigInterpreter().getPigServer();
|
||||
maxResult = Integer.parseInt(getProperty("zeppelin.pig.maxResult"));
|
||||
maxResult = Integer.parseInt(getProperty(MAX_RESULTS));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -104,7 +105,7 @@ public class PigQueryInterpreter extends BasePigInterpreter {
|
|||
Iterator<Tuple> iter = pigServer.openIterator(alias);
|
||||
boolean firstRow = true;
|
||||
int index = 0;
|
||||
while (iter.hasNext() && index <= maxResult) {
|
||||
while (iter.hasNext() && index < maxResult) {
|
||||
index++;
|
||||
Tuple tuple = iter.next();
|
||||
if (firstRow && !schemaKnown) {
|
||||
|
|
@ -118,7 +119,8 @@ public class PigQueryInterpreter extends BasePigInterpreter {
|
|||
resultBuilder.append("\n");
|
||||
}
|
||||
if (index >= maxResult && iter.hasNext()) {
|
||||
resultBuilder.append("\n<font color=red>Results are limited by " + maxResult + ".</font>");
|
||||
resultBuilder.append("\n");
|
||||
resultBuilder.append(ResultMessages.getExceedsLimitRowsMessage(maxResult, MAX_RESULTS));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// Extract error in the following order
|
||||
|
|
|
|||
|
|
@ -153,6 +153,6 @@ public class PigQueryInterpreterTest {
|
|||
assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType());
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertTrue(result.message().get(0).getData().contains("id\n0\n1\n2"));
|
||||
assertTrue(result.message().get(0).getData().contains("Results are limited by 20"));
|
||||
assertTrue(result.message().get(1).getData().contains("alert-warning"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
111
pom.xml
111
pom.xml
|
|
@ -56,6 +56,7 @@
|
|||
<module>zeppelin-zengine</module>
|
||||
<module>zeppelin-display</module>
|
||||
<module>spark-dependencies</module>
|
||||
<module>groovy</module>
|
||||
<module>spark</module>
|
||||
<module>markdown</module>
|
||||
<module>angular</module>
|
||||
|
|
@ -92,10 +93,12 @@
|
|||
<log4j.version>1.2.17</log4j.version>
|
||||
<libthrift.version>0.9.2</libthrift.version>
|
||||
<gson.version>2.2</gson.version>
|
||||
<gson-extras.version>0.2.1</gson-extras.version>
|
||||
<guava.version>15.0</guava.version>
|
||||
<jetty.version>9.2.15.v20160210</jetty.version>
|
||||
<httpcomponents.core.version>4.3.3</httpcomponents.core.version>
|
||||
<httpcomponents.client.version>4.3.6</httpcomponents.client.version>
|
||||
<httpcomponents.asyncclient.version>4.0.2</httpcomponents.asyncclient.version>
|
||||
<commons.lang.version>2.5</commons.lang.version>
|
||||
<commons.configuration.version>1.9</commons.configuration.version>
|
||||
<commons.codec.version>1.5</commons.codec.version>
|
||||
|
|
@ -172,6 +175,12 @@
|
|||
<version>${httpcomponents.client.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpasyncclient</artifactId>
|
||||
<version>${httpcomponents.asyncclient.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-lang</groupId>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
|
|
@ -184,6 +193,12 @@
|
|||
<version>${gson.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.danilopianini</groupId>
|
||||
<artifactId>gson-extras</artifactId>
|
||||
<version>${gson-extras.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-configuration</groupId>
|
||||
<artifactId>commons-configuration</artifactId>
|
||||
|
|
@ -924,39 +939,39 @@
|
|||
<exclude>**/interpreter-setting.json</exclude>
|
||||
<exclude>**/constants.json</exclude>
|
||||
<exclude>scripts/**</exclude>
|
||||
<exclude>**/**/*.log</exclude>
|
||||
<exclude>**/**/logs/**</exclude>
|
||||
<exclude>**/**/*.log</exclude>
|
||||
<exclude>**/**/logs/**</exclude>
|
||||
|
||||
<!-- bundled from zeppelin-web -->
|
||||
<exclude>**/test/karma.conf.js</exclude>
|
||||
<exclude>**/test/spec/**</exclude>
|
||||
<exclude>**/.babelrc</exclude>
|
||||
<exclude>**/.bowerrc</exclude>
|
||||
<exclude>.editorconfig</exclude>
|
||||
<exclude>.eslintrc</exclude>
|
||||
<exclude>**/.tmp/**</exclude>
|
||||
<exclude>**/target/**</exclude>
|
||||
<exclude>**/node/**</exclude>
|
||||
<exclude>**/node_modules/**</exclude>
|
||||
<exclude>**/bower_components/**</exclude>
|
||||
<exclude>**/dist/**</exclude>
|
||||
<exclude>**/.buildignore</exclude>
|
||||
<exclude>**/.npmignore</exclude>
|
||||
<exclude>**/.jshintrc</exclude>
|
||||
<exclude>**/yarn.lock</exclude>
|
||||
<exclude>**/bower.json</exclude>
|
||||
<exclude>**/src/fonts/Patua-One*</exclude>
|
||||
<exclude>**/src/fonts/patua-one*</exclude>
|
||||
<exclude>**/src/fonts/Roboto*</exclude>
|
||||
<exclude>**/src/fonts/roboto*</exclude>
|
||||
<exclude>**/src/fonts/fontawesome*</exclude>
|
||||
<exclude>**/src/fonts/font-awesome*</exclude>
|
||||
<exclude>**/src/styles/font-awesome*</exclude>
|
||||
<exclude>**/src/fonts/Simple-Line*</exclude>
|
||||
<exclude>**/src/fonts/simple-line*</exclude>
|
||||
<exclude>**/src/fonts/Source-Code-Pro*</exclude>
|
||||
<!-- bundled from zeppelin-web -->
|
||||
<exclude>**/test/karma.conf.js</exclude>
|
||||
<exclude>**/test/spec/**</exclude>
|
||||
<exclude>**/.babelrc</exclude>
|
||||
<exclude>**/.bowerrc</exclude>
|
||||
<exclude>.editorconfig</exclude>
|
||||
<exclude>.eslintrc</exclude>
|
||||
<exclude>**/.tmp/**</exclude>
|
||||
<exclude>**/target/**</exclude>
|
||||
<exclude>**/node/**</exclude>
|
||||
<exclude>**/node_modules/**</exclude>
|
||||
<exclude>**/bower_components/**</exclude>
|
||||
<exclude>**/dist/**</exclude>
|
||||
<exclude>**/.buildignore</exclude>
|
||||
<exclude>**/.npmignore</exclude>
|
||||
<exclude>**/.jshintrc</exclude>
|
||||
<exclude>**/yarn.lock</exclude>
|
||||
<exclude>**/bower.json</exclude>
|
||||
<exclude>**/src/fonts/Patua-One*</exclude>
|
||||
<exclude>**/src/fonts/patua-one*</exclude>
|
||||
<exclude>**/src/fonts/Roboto*</exclude>
|
||||
<exclude>**/src/fonts/roboto*</exclude>
|
||||
<exclude>**/src/fonts/fontawesome*</exclude>
|
||||
<exclude>**/src/fonts/font-awesome*</exclude>
|
||||
<exclude>**/src/styles/font-awesome*</exclude>
|
||||
<exclude>**/src/fonts/Simple-Line*</exclude>
|
||||
<exclude>**/src/fonts/simple-line*</exclude>
|
||||
<exclude>**/src/fonts/Source-Code-Pro*</exclude>
|
||||
<exclude>**/src/fonts/source-code-pro*</exclude>
|
||||
<exclude>**/src/**/**.test.js</exclude>
|
||||
<exclude>**/src/**/**.test.js</exclude>
|
||||
|
||||
<!-- from SQLLine 1.0.2, see ZEPPELIN-2135 -->
|
||||
<exclude>**/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java</exclude>
|
||||
|
|
@ -1006,24 +1021,24 @@
|
|||
<exclude>**/package.json</exclude>
|
||||
|
||||
<!-- compiled R packages (binaries) -->
|
||||
<exclude>**/R/lib/**</exclude>
|
||||
<exclude>**/r/lib/**</exclude>
|
||||
|
||||
<!--R-related files with alternative licenses-->
|
||||
<exclude>**/R/lib/**</exclude>
|
||||
<exclude>**/r/lib/**</exclude>
|
||||
|
||||
<exclude>**/R/rzeppelin/R/globals.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/common.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/protocol.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/rServer.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/scalaInterpreter.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/zzz.R</exclude>
|
||||
<exclude>**/src/main/scala/scala/Console.scala</exclude>
|
||||
<exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
|
||||
<exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
|
||||
|
||||
<!--The following files are mechanical-->
|
||||
<exclude>**/R/rzeppelin/DESCRIPTION</exclude>
|
||||
<exclude>**/R/rzeppelin/NAMESPACE</exclude>
|
||||
<!--R-related files with alternative licenses-->
|
||||
|
||||
<exclude>**/R/rzeppelin/R/globals.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/common.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/protocol.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/rServer.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/scalaInterpreter.R</exclude>
|
||||
<exclude>**/R/rzeppelin/R/zzz.R</exclude>
|
||||
<exclude>**/src/main/scala/scala/Console.scala</exclude>
|
||||
<exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/Package.scala</exclude>
|
||||
<exclude>**/src/main/scala/org/apache/zeppelin/rinterpreter/rscala/RClient.scala</exclude>
|
||||
|
||||
<!--The following files are mechanical-->
|
||||
<exclude>**/R/rzeppelin/DESCRIPTION</exclude>
|
||||
<exclude>**/R/rzeppelin/NAMESPACE</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@
|
|||
<name>Zeppelin: Python interpreter</name>
|
||||
|
||||
<properties>
|
||||
<py4j.version>0.9.2</py4j.version>
|
||||
<python.py4j.version>0.9.2</python.py4j.version>
|
||||
<python.test.exclude>
|
||||
**/PythonInterpreterWithPythonInstalledTest.java,
|
||||
**/PythonInterpreterPandasSqlTest.java,
|
||||
|
|
@ -58,7 +58,7 @@
|
|||
<dependency>
|
||||
<groupId>net.sf.py4j</groupId>
|
||||
<artifactId>py4j</artifactId>
|
||||
<version>${py4j.version}</version>
|
||||
<version>${python.py4j.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
@ -108,8 +108,8 @@
|
|||
<goals><goal>download-single</goal></goals>
|
||||
<configuration>
|
||||
<url>https://pypi.python.org/packages/64/5c/01e13b68e8caafece40d549f232c9b5677ad1016071a48d04cc3895acaa3</url>
|
||||
<fromFile>py4j-${py4j.version}.zip</fromFile>
|
||||
<toFile>${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip</toFile>
|
||||
<fromFile>py4j-${python.py4j.version}.zip</fromFile>
|
||||
<toFile>${project.build.directory}/../../interpreter/python/py4j-${python.py4j.version}.zip</toFile>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
|
|
@ -123,7 +123,7 @@
|
|||
<phase>package</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<unzip src="${project.build.directory}/../../interpreter/python/py4j-${py4j.version}.zip"
|
||||
<unzip src="${project.build.directory}/../../interpreter/python/py4j-${python.py4j.version}.zip"
|
||||
dest="${project.build.directory}/../../interpreter/python"/>
|
||||
</target>
|
||||
</configuration>
|
||||
|
|
|
|||
|
|
@ -222,7 +222,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
|
|||
// Add matplotlib display hook
|
||||
InterpreterGroup intpGroup = getInterpreterGroup();
|
||||
if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
|
||||
registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
|
||||
registerHook(HookType.POST_EXEC_DEV, "__zeppelin__._displayhook()");
|
||||
}
|
||||
// Add matplotlib display hook
|
||||
try {
|
||||
|
|
@ -435,7 +435,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -87,7 +87,8 @@ public class PythonInterpreterPandasSql extends Interpreter {
|
|||
LOG.info("Running SQL query: '{}' over Pandas DataFrame", st);
|
||||
Interpreter python = getPythonInterpreter();
|
||||
|
||||
return python.interpret("z.show(pysqldf('" + st + "'))\nz._displayhook()", context);
|
||||
return python.interpret(
|
||||
"__zeppelin__.show(pysqldf('" + st + "'))\n__zeppelin__._displayhook()", context);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ class PyZeppelinContext(object):
|
|||
|
||||
def __init__(self, z):
|
||||
self.z = z
|
||||
self.paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
|
||||
self.paramOption = gateway.jvm.org.apache.zeppelin.display.ui.OptionInput.ParamOption
|
||||
self.javaList = gateway.jvm.java.util.ArrayList
|
||||
self.max_result = 1000
|
||||
self._displayhook = lambda *args: None
|
||||
|
|
@ -195,6 +195,7 @@ host = "127.0.0.1"
|
|||
if len(sys.argv) >= 3:
|
||||
host = sys.argv[2]
|
||||
|
||||
_zcUserQueryNameSpace = {}
|
||||
client = GatewayClient(address=host, port=int(sys.argv[1]))
|
||||
|
||||
#gateway = JavaGateway(client, auto_convert = True)
|
||||
|
|
@ -204,8 +205,11 @@ intp = gateway.entry_point
|
|||
intp.onPythonScriptInitialized(os.getpid())
|
||||
|
||||
java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
|
||||
z = PyZeppelinContext(intp)
|
||||
z._setup_matplotlib()
|
||||
z = __zeppelin__ = PyZeppelinContext(intp)
|
||||
__zeppelin__._setup_matplotlib()
|
||||
|
||||
_zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__
|
||||
_zcUserQueryNameSpace["z"] = z
|
||||
|
||||
output = Logger()
|
||||
sys.stdout = output
|
||||
|
|
@ -227,7 +231,7 @@ while True :
|
|||
global_hook = None
|
||||
|
||||
try:
|
||||
user_hook = z.getHook('post_exec')
|
||||
user_hook = __zeppelin__.getHook('post_exec')
|
||||
except:
|
||||
user_hook = None
|
||||
|
||||
|
|
@ -263,17 +267,17 @@ while True :
|
|||
for node in to_run_exec:
|
||||
mod = ast.Module([node])
|
||||
code = compile(mod, '<stdin>', 'exec')
|
||||
exec(code)
|
||||
exec(code, _zcUserQueryNameSpace)
|
||||
|
||||
for node in to_run_single:
|
||||
mod = ast.Interactive([node])
|
||||
code = compile(mod, '<stdin>', 'single')
|
||||
exec(code)
|
||||
exec(code, _zcUserQueryNameSpace)
|
||||
|
||||
for node in to_run_hooks:
|
||||
mod = ast.Module([node])
|
||||
code = compile(mod, '<stdin>', 'exec')
|
||||
exec(code)
|
||||
exec(code, _zcUserQueryNameSpace)
|
||||
except:
|
||||
raise Exception(traceback.format_exc())
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,19 @@ public class PythonInterpreterTest implements InterpreterOutputListener {
|
|||
assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("hi\nhi\nhi"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRedefinitionZeppelinContext() {
|
||||
String pyRedefinitionCode = "z = 1\n";
|
||||
String pyRestoreCode = "z = __zeppelin__\n";
|
||||
String pyValidCode = "z.input(\"test\")\n";
|
||||
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyValidCode, context).code());
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyRedefinitionCode, context).code());
|
||||
assertEquals(InterpreterResult.Code.ERROR, pythonInterpreter.interpret(pyValidCode, context).code());
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyRestoreCode, context).code());
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, pythonInterpreter.interpret(pyValidCode, context).code());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onUpdateAll(InterpreterOutput out) {
|
||||
|
||||
|
|
|
|||
|
|
@ -77,8 +77,9 @@ public class KnitR extends Interpreter implements WrappedInterpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String s, int i) {
|
||||
List completion = intp.completion(s, i);
|
||||
public List<InterpreterCompletion> completion(String s, int i,
|
||||
InterpreterContext interpreterContext) {
|
||||
List completion = intp.completion(s, i, interpreterContext);
|
||||
return completion;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -77,8 +77,9 @@ public class RRepl extends Interpreter implements WrappedInterpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String s, int i) {
|
||||
List completion = intp.completion(s, i);
|
||||
public List<InterpreterCompletion> completion(String s, int i,
|
||||
InterpreterContext interpreterContext) {
|
||||
List completion = intp.completion(s, i, interpreterContext);
|
||||
return completion;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -270,7 +270,8 @@ public class ScaldingInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return NO_COMPLETION;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ This assumes you've already cloned the project either on the host machine in the
|
|||
|
||||
```
|
||||
cd /zeppelin
|
||||
mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests
|
||||
mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests
|
||||
./bin/zeppelin-daemon.sh start
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ echo 'mvn clean package -DskipTests'
|
|||
echo
|
||||
echo '# or for a specific Spark/Hadoop build with additional options such as python and R support'
|
||||
echo
|
||||
echo 'mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests'
|
||||
echo 'mvn clean package -Pspark-1.6 -Phadoop-2.4 -DskipTests'
|
||||
echo './bin/zeppelin-daemon.sh start'
|
||||
echo
|
||||
echo 'On your host machine browse to http://localhost:8080/'
|
||||
|
|
|
|||
|
|
@ -138,7 +138,8 @@ public class ShellInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@
|
|||
<spark.bin.download.url>
|
||||
http://d3kbcqa49mib13.cloudfront.net/spark-${spark.version}-bin-without-hadoop.tgz
|
||||
</spark.bin.download.url>
|
||||
<py4j.version>0.8.2.1</py4j.version>
|
||||
<spark.py4j.version>0.8.2.1</spark.py4j.version>
|
||||
|
||||
<!--plugin versions-->
|
||||
<plugin.shade.version>2.3</plugin.shade.version>
|
||||
|
|
@ -344,6 +344,19 @@
|
|||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- yarn (not supported for Spark v1.5.0 or higher) -->
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
<version>${yarn.version}</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<profiles>
|
||||
|
|
@ -514,7 +527,7 @@
|
|||
<id>spark-1.6</id>
|
||||
<properties>
|
||||
<spark.version>1.6.3</spark.version>
|
||||
<py4j.version>0.9</py4j.version>
|
||||
<spark.py4j.version>0.9</spark.py4j.version>
|
||||
<akka.group>com.typesafe.akka</akka.group>
|
||||
<akka.version>2.3.11</akka.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
|
|
@ -526,7 +539,7 @@
|
|||
<properties>
|
||||
<spark.version>2.0.2</spark.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<py4j.version>0.10.3</py4j.version>
|
||||
<spark.py4j.version>0.10.3</spark.py4j.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
@ -539,7 +552,7 @@
|
|||
<properties>
|
||||
<spark.version>2.1.0</spark.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<py4j.version>0.10.4</py4j.version>
|
||||
<spark.py4j.version>0.10.4</spark.py4j.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
@ -810,137 +823,6 @@
|
|||
</repositories>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>yarn</id>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.spark</groupId>
|
||||
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
|
||||
<version>${spark.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
<version>${yarn.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>pyspark</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>com.googlecode.maven-download-plugin</groupId>
|
||||
<artifactId>download-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>download-pyspark-files</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>wget</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<readTimeOut>60000</readTimeOut>
|
||||
<retries>5</retries>
|
||||
<unpack>true</unpack>
|
||||
<url>${spark.src.download.url}</url>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-clean-plugin</artifactId>
|
||||
<configuration>
|
||||
<filesets>
|
||||
<fileset>
|
||||
<directory>${basedir}/../python/build</directory>
|
||||
</fileset>
|
||||
</filesets>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>zip-pyspark-files</id>
|
||||
<phase>generate-resources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<delete dir="../interpreter/spark/pyspark"/>
|
||||
<copy todir="../interpreter/spark/pyspark"
|
||||
file="${project.build.directory}/${spark.archive}/python/lib/py4j-${py4j.version}-src.zip"/>
|
||||
<zip destfile="${project.build.directory}/../../interpreter/spark/pyspark/pyspark.zip"
|
||||
basedir="${project.build.directory}/${spark.archive}/python"
|
||||
includes="pyspark/*.py,pyspark/**/*.py"/>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
|
||||
<profile>
|
||||
<id>sparkr</id>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>com.googlecode.maven-download-plugin</groupId>
|
||||
<artifactId>download-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>download-sparkr-files</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>wget</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<readTimeOut>60000</readTimeOut>
|
||||
<retries>5</retries>
|
||||
<url>${spark.bin.download.url}</url>
|
||||
<unpack>true</unpack>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-resources-plugin</artifactId>
|
||||
<version>2.7</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-sparkr-files</id>
|
||||
<phase>generate-resources</phase>
|
||||
<goals>
|
||||
<goal>copy-resources</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/spark/R/lib</outputDirectory>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>
|
||||
${project.build.directory}/spark-${spark.version}-bin-without-hadoop/R/lib
|
||||
</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
|
||||
</profiles>
|
||||
|
||||
<build>
|
||||
|
|
@ -1045,6 +927,108 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<!-- include pyspark by default -->
|
||||
<plugin>
|
||||
<groupId>com.googlecode.maven-download-plugin</groupId>
|
||||
<artifactId>download-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>download-pyspark-files</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>wget</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<readTimeOut>60000</readTimeOut>
|
||||
<retries>5</retries>
|
||||
<unpack>true</unpack>
|
||||
<url>${spark.src.download.url}</url>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-clean-plugin</artifactId>
|
||||
<configuration>
|
||||
<filesets>
|
||||
<fileset>
|
||||
<directory>${basedir}/../python/build</directory>
|
||||
</fileset>
|
||||
</filesets>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>zip-pyspark-files</id>
|
||||
<phase>generate-resources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<delete dir="../interpreter/spark/pyspark"/>
|
||||
<copy todir="../interpreter/spark/pyspark"
|
||||
file="${project.build.directory}/${spark.archive}/python/lib/py4j-${spark.py4j.version}-src.zip"/>
|
||||
<zip destfile="${project.build.directory}/../../interpreter/spark/pyspark/pyspark.zip"
|
||||
basedir="${project.build.directory}/${spark.archive}/python"
|
||||
includes="pyspark/*.py,pyspark/**/*.py"/>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
|
||||
<!-- include sparkr by default -->
|
||||
<plugin>
|
||||
<groupId>com.googlecode.maven-download-plugin</groupId>
|
||||
<artifactId>download-maven-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>download-sparkr-files</id>
|
||||
<phase>validate</phase>
|
||||
<goals>
|
||||
<goal>wget</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<readTimeOut>60000</readTimeOut>
|
||||
<retries>5</retries>
|
||||
<url>${spark.bin.download.url}</url>
|
||||
<unpack>true</unpack>
|
||||
<outputDirectory>${project.build.directory}</outputDirectory>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-resources-plugin</artifactId>
|
||||
<version>2.7</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>copy-sparkr-files</id>
|
||||
<phase>generate-resources</phase>
|
||||
<goals>
|
||||
<goal>copy-resources</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<outputDirectory>${project.build.directory}/../../interpreter/spark/R/lib</outputDirectory>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>
|
||||
${project.build.directory}/spark-${spark.version}-bin-without-hadoop/R/lib
|
||||
</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
|
|
|||
115
spark/pom.xml
115
spark/pom.xml
|
|
@ -320,6 +320,19 @@
|
|||
</dependencies>
|
||||
|
||||
<build>
|
||||
<!-- sparkr resources -->
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<excludes>
|
||||
<exclude>interpreter-setting.json</exclude>
|
||||
</excludes>
|
||||
</resource>
|
||||
<resource>
|
||||
<directory>src/main/sparkr-resources</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-enforcer-plugin</artifactId>
|
||||
|
|
@ -440,18 +453,13 @@
|
|||
</executions>
|
||||
</plugin>
|
||||
|
||||
<!-- exclude sparkr by default. sparkr is enabled by profile 'sparkr' -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>**/SparkRInterpreter.java</exclude>
|
||||
</excludes>
|
||||
<testExcludes>
|
||||
<excludes combine.self="override"></excludes>
|
||||
<testExcludes combine.self="override">
|
||||
<testExclude>${pyspark.test.exclude}</testExclude>
|
||||
<testExclude>**/SparkRInterpreterTest.java</testExclude>
|
||||
<testExclude>**/ZeppelinRTest.java</testExclude>
|
||||
</testExcludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
|
@ -459,9 +467,7 @@
|
|||
<groupId>org.scala-tools</groupId>
|
||||
<artifactId>maven-scala-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>**/ZeppelinR.scala</exclude>
|
||||
<exclude>**/SparkRBackend.scala</exclude>
|
||||
<excludes combine.self="override">
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
|
@ -469,12 +475,37 @@
|
|||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<includes>
|
||||
<include>${pyspark.test.include}</include>
|
||||
</includes>
|
||||
<excludes>
|
||||
<excludes combine.self="override">
|
||||
<exclude>${pyspark.test.exclude}</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<!-- include sparkr by default -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes combine.self="override"></excludes>
|
||||
<testExcludes combine.self="override">
|
||||
<testExclude>${pyspark.test.exclude}</testExclude>
|
||||
</testExcludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.scala-tools</groupId>
|
||||
<artifactId>maven-scala-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes combine.self="override">
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes combine.self="override">
|
||||
<exclude>${pyspark.test.exclude}</exclude>
|
||||
<exclude>**/SparkRInterpreterTest.java</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
|
@ -506,7 +537,7 @@
|
|||
<id>spark-1.6</id>
|
||||
<properties>
|
||||
<spark.version>1.6.3</spark.version>
|
||||
<py4j.version>0.9</py4j.version>
|
||||
<spark.py4j.version>0.9</spark.py4j.version>
|
||||
<akka.group>com.typesafe.akka</akka.group>
|
||||
<akka.version>2.3.11</akka.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
|
|
@ -518,7 +549,7 @@
|
|||
<properties>
|
||||
<spark.version>2.0.2</spark.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<py4j.version>0.10.3</py4j.version>
|
||||
<spark.py4j.version>0.10.3</spark.py4j.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
@ -531,7 +562,7 @@
|
|||
<properties>
|
||||
<spark.version>2.1.0</spark.version>
|
||||
<protobuf.version>2.5.0</protobuf.version>
|
||||
<py4j.version>0.10.4</py4j.version>
|
||||
<spark.py4j.version>0.10.4</spark.py4j.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
</properties>
|
||||
</profile>
|
||||
|
|
@ -609,53 +640,5 @@
|
|||
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<!-- include sparkr in the build -->
|
||||
<profile>
|
||||
<id>sparkr</id>
|
||||
<build>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<excludes>
|
||||
<exclude>interpreter-setting.json</exclude>
|
||||
</excludes>
|
||||
</resource>
|
||||
<resource>
|
||||
<directory>src/main/sparkr-resources</directory>
|
||||
</resource>
|
||||
</resources>
|
||||
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes combine.self="override"></excludes>
|
||||
<testExcludes combine.self="override">
|
||||
<testExclude>${pyspark.test.exclude}</testExclude>
|
||||
</testExcludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.scala-tools</groupId>
|
||||
<artifactId>maven-scala-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes combine.self="override">
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes combine.self="override">
|
||||
<exclude>${pyspark.test.exclude}</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ import java.util.Properties;
|
|||
|
||||
import com.google.common.reflect.TypeToken;
|
||||
import com.google.gson.Gson;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.spark.repl.SparkILoop;
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
|
|
@ -284,7 +286,8 @@ public class DepInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
if (Utils.isScala2_10()) {
|
||||
ScalaCompleter c = (ScalaCompleter) Utils.invokeMethod(completer, "completer");
|
||||
Candidates ret = c.complete(buf, cursor);
|
||||
|
|
@ -293,7 +296,7 @@ public class DepInterpreter extends Interpreter {
|
|||
List<InterpreterCompletion> completions = new LinkedList<>();
|
||||
|
||||
for (String candidate : candidates) {
|
||||
completions.add(new InterpreterCompletion(candidate, candidate));
|
||||
completions.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY));
|
||||
}
|
||||
|
||||
return completions;
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import org.apache.commons.exec.ExecuteResultHandler;
|
|||
import org.apache.commons.exec.ExecuteWatchdog;
|
||||
import org.apache.commons.exec.PumpStreamHandler;
|
||||
import org.apache.commons.exec.environment.EnvironmentUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.spark.SparkConf;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
import org.apache.spark.sql.SQLContext;
|
||||
|
|
@ -113,7 +114,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
// Add matplotlib display hook
|
||||
InterpreterGroup intpGroup = getInterpreterGroup();
|
||||
if (intpGroup != null && intpGroup.getInterpreterHookRegistry() != null) {
|
||||
registerHook(HookType.POST_EXEC_DEV, "z._displayhook()");
|
||||
registerHook(HookType.POST_EXEC_DEV, "__zeppelin__._displayhook()");
|
||||
}
|
||||
DepInterpreter depInterpreter = getDepInterpreter();
|
||||
|
||||
|
|
@ -337,7 +338,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
public InterpreterResult interpret(String st, InterpreterContext context) {
|
||||
SparkInterpreter sparkInterpreter = getSparkInterpreter();
|
||||
sparkInterpreter.populateSparkWebUrl(context);
|
||||
if (sparkInterpreter.getSparkVersion().isUnsupportedVersion()) {
|
||||
if (sparkInterpreter.isUnsupportedSparkVersion()) {
|
||||
return new InterpreterResult(Code.ERROR, "Spark "
|
||||
+ sparkInterpreter.getSparkVersion().toString() + " is not supported");
|
||||
}
|
||||
|
|
@ -390,9 +391,9 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
return new InterpreterResult(Code.ERROR, errorMessage);
|
||||
}
|
||||
String jobGroup = Utils.buildJobGroupId(context);
|
||||
ZeppelinContext z = sparkInterpreter.getZeppelinContext();
|
||||
z.setInterpreterContext(context);
|
||||
z.setGui(context.getGui());
|
||||
ZeppelinContext __zeppelin__ = sparkInterpreter.getZeppelinContext();
|
||||
__zeppelin__.setInterpreterContext(context);
|
||||
__zeppelin__.setGui(context.getGui());
|
||||
pythonInterpretRequest = new PythonInterpretRequest(st, jobGroup);
|
||||
statementOutput = null;
|
||||
|
||||
|
|
@ -457,7 +458,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
if (buf.length() < cursor) {
|
||||
cursor = buf.length();
|
||||
}
|
||||
|
|
@ -466,8 +468,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
|
||||
//start code for completion
|
||||
SparkInterpreter sparkInterpreter = getSparkInterpreter();
|
||||
if (sparkInterpreter.getSparkVersion().isUnsupportedVersion() == false
|
||||
&& pythonscriptRunning == false) {
|
||||
if (sparkInterpreter.isUnsupportedSparkVersion() || pythonscriptRunning == false) {
|
||||
return new LinkedList<>();
|
||||
}
|
||||
|
||||
|
|
@ -509,7 +510,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
|
||||
List<InterpreterCompletion> results = new LinkedList<>();
|
||||
for (String name: completionList) {
|
||||
results.add(new InterpreterCompletion(name, name));
|
||||
results.add(new InterpreterCompletion(name, name, StringUtils.EMPTY));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ import org.apache.spark.repl.SparkILoop;
|
|||
import org.apache.spark.scheduler.ActiveJob;
|
||||
import org.apache.spark.scheduler.DAGScheduler;
|
||||
import org.apache.spark.scheduler.Pool;
|
||||
import org.apache.spark.scheduler.SparkListenerApplicationEnd;
|
||||
import org.apache.spark.scheduler.SparkListenerJobStart;
|
||||
import org.apache.spark.sql.SQLContext;
|
||||
import org.apache.spark.ui.SparkUI;
|
||||
|
|
@ -128,7 +127,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
private static File outputDir; // class outputdir for scala 2.11
|
||||
private Object classServer; // classserver for scala 2.11
|
||||
private JavaSparkContext jsc;
|
||||
|
||||
private boolean enableSupportedVersionCheck;
|
||||
|
||||
public SparkInterpreter(Properties property) {
|
||||
super(property);
|
||||
|
|
@ -247,7 +246,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
*/
|
||||
private boolean hiveClassesArePresent() {
|
||||
try {
|
||||
this.getClass().forName("org.apache.spark.sql.hive.HiveSessionState");
|
||||
this.getClass().forName("org.apache.spark.sql.hive.execution.InsertIntoHiveTable");
|
||||
this.getClass().forName("org.apache.hadoop.hive.conf.HiveConf");
|
||||
return true;
|
||||
} catch (ClassNotFoundException | NoClassDefFoundError e) {
|
||||
|
|
@ -609,6 +608,9 @@ public class SparkInterpreter extends Interpreter {
|
|||
|
||||
@Override
|
||||
public void open() {
|
||||
this.enableSupportedVersionCheck = java.lang.Boolean.parseBoolean(
|
||||
property.getProperty("zeppelin.spark.enableSupportedVersionCheck", "true"));
|
||||
|
||||
// set properties and do login before creating any spark stuff for secured cluster
|
||||
if (isYarnMode()) {
|
||||
System.setProperty("SPARK_YARN_MODE", "true");
|
||||
|
|
@ -997,10 +999,19 @@ public class SparkInterpreter extends Interpreter {
|
|||
if (sparkUrl != null) {
|
||||
return sparkUrl;
|
||||
}
|
||||
Option<SparkUI> sparkUiOption = (Option<SparkUI>) Utils.invokeMethod(sc, "ui");
|
||||
SparkUI sparkUi = sparkUiOption.get();
|
||||
String sparkWebUrl = sparkUi.appUIAddress();
|
||||
return sparkWebUrl;
|
||||
|
||||
if (sparkVersion.newerThanEquals(SparkVersion.SPARK_2_0_0)) {
|
||||
Option<String> uiWebUrlOption = (Option<String>) Utils.invokeMethod(sc, "uiWebUrl");
|
||||
if (uiWebUrlOption.isDefined()) {
|
||||
return uiWebUrlOption.get();
|
||||
}
|
||||
} else {
|
||||
Option<SparkUI> sparkUIOption = (Option<SparkUI>) Utils.invokeMethod(sc, "ui");
|
||||
if (sparkUIOption.isDefined()) {
|
||||
return (String) Utils.invokeMethod(sparkUIOption.get(), "appUIAddress");
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private Results.Result interpret(String line) {
|
||||
|
|
@ -1057,7 +1068,8 @@ public class SparkInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
if (completer == null) {
|
||||
logger.warn("Can't find completer");
|
||||
return new LinkedList<>();
|
||||
|
|
@ -1079,7 +1091,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
List<InterpreterCompletion> completions = new LinkedList<>();
|
||||
|
||||
for (String candidate : candidates) {
|
||||
completions.add(new InterpreterCompletion(candidate, candidate));
|
||||
completions.add(new InterpreterCompletion(candidate, candidate, StringUtils.EMPTY));
|
||||
}
|
||||
|
||||
return completions;
|
||||
|
|
@ -1153,12 +1165,16 @@ public class SparkInterpreter extends Interpreter {
|
|||
return obj;
|
||||
}
|
||||
|
||||
boolean isUnsupportedSparkVersion() {
|
||||
return enableSupportedVersionCheck && sparkVersion.isUnsupportedVersion();
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpret a single line.
|
||||
*/
|
||||
@Override
|
||||
public InterpreterResult interpret(String line, InterpreterContext context) {
|
||||
if (sparkVersion.isUnsupportedVersion()) {
|
||||
if (isUnsupportedSparkVersion()) {
|
||||
return new InterpreterResult(Code.ERROR, "Spark " + sparkVersion.toString()
|
||||
+ " is not supported");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -108,6 +108,10 @@ public class SparkRInterpreter extends Interpreter {
|
|||
|
||||
SparkInterpreter sparkInterpreter = getSparkInterpreter();
|
||||
sparkInterpreter.populateSparkWebUrl(interpreterContext);
|
||||
if (sparkInterpreter.isUnsupportedSparkVersion()) {
|
||||
return new InterpreterResult(InterpreterResult.Code.ERROR, "Spark "
|
||||
+ sparkInterpreter.getSparkVersion().toString() + " is not supported");
|
||||
}
|
||||
|
||||
String jobGroup = Utils.buildJobGroupId(interpreterContext);
|
||||
sparkInterpreter.getSparkContext().setJobGroup(jobGroup, "Zeppelin", false);
|
||||
|
|
@ -208,7 +212,8 @@ public class SparkRInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,10 @@ import org.slf4j.LoggerFactory;
|
|||
* Spark SQL interpreter for Zeppelin.
|
||||
*/
|
||||
public class SparkSqlInterpreter extends Interpreter {
|
||||
Logger logger = LoggerFactory.getLogger(SparkSqlInterpreter.class);
|
||||
private Logger logger = LoggerFactory.getLogger(SparkSqlInterpreter.class);
|
||||
|
||||
public static final String MAX_RESULTS = "zeppelin.spark.maxResult";
|
||||
|
||||
AtomicInteger num = new AtomicInteger(0);
|
||||
|
||||
private int maxResult;
|
||||
|
|
@ -53,7 +56,7 @@ public class SparkSqlInterpreter extends Interpreter {
|
|||
|
||||
@Override
|
||||
public void open() {
|
||||
this.maxResult = Integer.parseInt(getProperty("zeppelin.spark.maxResult"));
|
||||
this.maxResult = Integer.parseInt(getProperty(MAX_RESULTS));
|
||||
}
|
||||
|
||||
private SparkInterpreter getSparkInterpreter() {
|
||||
|
|
@ -87,7 +90,7 @@ public class SparkSqlInterpreter extends Interpreter {
|
|||
SQLContext sqlc = null;
|
||||
SparkInterpreter sparkInterpreter = getSparkInterpreter();
|
||||
|
||||
if (sparkInterpreter.getSparkVersion().isUnsupportedVersion()) {
|
||||
if (sparkInterpreter.isUnsupportedSparkVersion()) {
|
||||
return new InterpreterResult(Code.ERROR, "Spark "
|
||||
+ sparkInterpreter.getSparkVersion().toString() + " is not supported");
|
||||
}
|
||||
|
|
@ -174,7 +177,8 @@ public class SparkSqlInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor) {
|
||||
public List<InterpreterCompletion> completion(String buf, int cursor,
|
||||
InterpreterContext interpreterContext) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,12 +40,13 @@ import org.apache.zeppelin.display.AngularObject;
|
|||
import org.apache.zeppelin.display.AngularObjectRegistry;
|
||||
import org.apache.zeppelin.display.AngularObjectWatcher;
|
||||
import org.apache.zeppelin.display.GUI;
|
||||
import org.apache.zeppelin.display.Input.ParamOption;
|
||||
import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
|
||||
import org.apache.zeppelin.interpreter.InterpreterException;
|
||||
import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
|
||||
import org.apache.zeppelin.interpreter.RemoteWorksController;
|
||||
import org.apache.zeppelin.interpreter.ResultMessages;
|
||||
import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
|
||||
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
|
||||
import org.apache.zeppelin.resource.Resource;
|
||||
|
|
@ -113,14 +114,33 @@ public class ZeppelinContext {
|
|||
public SQLContext sqlContext;
|
||||
private GUI gui;
|
||||
|
||||
/**
|
||||
* @deprecated use z.textbox instead
|
||||
*
|
||||
*/
|
||||
@Deprecated
|
||||
@ZeppelinApi
|
||||
public Object input(String name) {
|
||||
return input(name, "");
|
||||
return textbox(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated use z.textbox instead
|
||||
*/
|
||||
@Deprecated
|
||||
@ZeppelinApi
|
||||
public Object input(String name, Object defaultValue) {
|
||||
return textbox(name, defaultValue.toString());
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object input(String name, Object defaultValue) {
|
||||
return gui.input(name, defaultValue);
|
||||
public Object textbox(String name) {
|
||||
return textbox(name, "");
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object textbox(String name, String defaultValue) {
|
||||
return gui.textbox(name, defaultValue);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
|
|
@ -135,7 +155,7 @@ public class ZeppelinContext {
|
|||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public scala.collection.Iterable<Object> checkbox(String name,
|
||||
public scala.collection.Seq<Object> checkbox(String name,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
List<Object> allChecked = new LinkedList<>();
|
||||
for (Tuple2<Object, String> option : asJavaIterable(options)) {
|
||||
|
|
@ -145,11 +165,12 @@ public class ZeppelinContext {
|
|||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public scala.collection.Iterable<Object> checkbox(String name,
|
||||
public scala.collection.Seq<Object> checkbox(String name,
|
||||
scala.collection.Iterable<Object> defaultChecked,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
return collectionAsScalaIterable(gui.checkbox(name, asJavaCollection(defaultChecked),
|
||||
tuplesToParamOptions(options)));
|
||||
return scala.collection.JavaConversions.asScalaBuffer(
|
||||
gui.checkbox(name, asJavaCollection(defaultChecked),
|
||||
tuplesToParamOptions(options))).toSeq();
|
||||
}
|
||||
|
||||
private ParamOption[] tuplesToParamOptions(
|
||||
|
|
@ -295,9 +316,9 @@ public class ZeppelinContext {
|
|||
}
|
||||
|
||||
if (rows.length > maxResult) {
|
||||
msg.append("<!--TABLE_COMMENT-->");
|
||||
msg.append("\n");
|
||||
msg.append("<font color=red>Results are limited by " + maxResult + ".</font>");
|
||||
msg.append(ResultMessages.getExceedsLimitRowsMessage(maxResult,
|
||||
SparkSqlInterpreter.MAX_RESULTS));
|
||||
}
|
||||
sc.clearJobGroup();
|
||||
return msg.toString();
|
||||
|
|
@ -310,7 +331,7 @@ public class ZeppelinContext {
|
|||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String noteId, String paragraphId) {
|
||||
run(noteId, paragraphId, interpreterContext);
|
||||
run(noteId, paragraphId, interpreterContext, true);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -319,8 +340,27 @@ public class ZeppelinContext {
|
|||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String paragraphId) {
|
||||
run(paragraphId, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph by id
|
||||
* @param paragraphId
|
||||
* @param checkCurrentParagraph
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String paragraphId, boolean checkCurrentParagraph) {
|
||||
String noteId = interpreterContext.getNoteId();
|
||||
run(noteId, paragraphId, interpreterContext);
|
||||
run(noteId, paragraphId, interpreterContext, checkCurrentParagraph);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph by id
|
||||
* @param noteId
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String noteId, String paragraphId, InterpreterContext context) {
|
||||
run(noteId, paragraphId, context, true);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -329,8 +369,9 @@ public class ZeppelinContext {
|
|||
* @param context
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void run(String noteId, String paragraphId, InterpreterContext context) {
|
||||
if (paragraphId.equals(context.getParagraphId())) {
|
||||
public void run(String noteId, String paragraphId, InterpreterContext context,
|
||||
boolean checkCurrentParagraph) {
|
||||
if (paragraphId.equals(context.getParagraphId()) && checkCurrentParagraph) {
|
||||
throw new InterpreterException("Can not run current Paragraph");
|
||||
}
|
||||
|
||||
|
|
@ -410,24 +451,50 @@ public class ZeppelinContext {
|
|||
*/
|
||||
@ZeppelinApi
|
||||
public void run(int idx) {
|
||||
run(idx, true);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param idx paragraph index
|
||||
* @param checkCurrentParagraph check whether you call this run method in the current paragraph.
|
||||
* Set it to false only when you are sure you are not invoking this method to run current
|
||||
* paragraph. Otherwise you would run current paragraph in infinite loop.
|
||||
*/
|
||||
public void run(int idx, boolean checkCurrentParagraph) {
|
||||
String noteId = interpreterContext.getNoteId();
|
||||
run(noteId, idx, interpreterContext);
|
||||
run(noteId, idx, interpreterContext, checkCurrentParagraph);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run paragraph at index
|
||||
* @param noteId
|
||||
* @param idx index starting from 0
|
||||
* @param context interpreter context
|
||||
*/
|
||||
public void run(String noteId, int idx, InterpreterContext context) {
|
||||
run(noteId, idx, context, true);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param noteId
|
||||
* @param idx paragraph index
|
||||
* @param context interpreter context
|
||||
* @param checkCurrentParagraph check whether you call this run method in the current paragraph.
|
||||
* Set it to false only when you are sure you are not invoking this method to run current
|
||||
* paragraph. Otherwise you would run current paragraph in infinite loop.
|
||||
*/
|
||||
public void run(String noteId, int idx, InterpreterContext context,
|
||||
boolean checkCurrentParagraph) {
|
||||
List<InterpreterContextRunner> runners = getInterpreterContextRunner(noteId, context);
|
||||
if (idx >= runners.size()) {
|
||||
throw new InterpreterException("Index out of bound");
|
||||
}
|
||||
|
||||
InterpreterContextRunner runner = runners.get(idx);
|
||||
if (runner.getParagraphId().equals(context.getParagraphId())) {
|
||||
throw new InterpreterException("Can not run current Paragraph");
|
||||
if (runner.getParagraphId().equals(context.getParagraphId()) && checkCurrentParagraph) {
|
||||
throw new InterpreterException("Can not run current Paragraph: " + runner.getParagraphId());
|
||||
}
|
||||
|
||||
runner.run();
|
||||
|
|
|
|||
|
|
@ -53,6 +53,12 @@
|
|||
"propertyName": "spark.master",
|
||||
"defaultValue": "local[*]",
|
||||
"description": "Spark master uri. ex) spark://masterhost:7077"
|
||||
},
|
||||
"zeppelin.spark.unSupportedVersionCheck": {
|
||||
"envName": null,
|
||||
"propertyName": "zeppelin.spark.enableSupportedVersionCheck",
|
||||
"defaultValue": "true",
|
||||
"description": "Do not change - developer only setting, not for production use"
|
||||
}
|
||||
},
|
||||
"editor": {
|
||||
|
|
|
|||
|
|
@ -97,13 +97,15 @@ class PyZeppelinContext(dict):
|
|||
|
||||
def checkbox(self, name, options, defaultChecked=None):
|
||||
if defaultChecked is None:
|
||||
defaultChecked = list(map(lambda items: items[0], options))
|
||||
defaultChecked = []
|
||||
optionTuples = list(map(lambda items: self.__tupleToScalaTuple2(items), options))
|
||||
optionIterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(optionTuples)
|
||||
defaultCheckedIterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(defaultChecked)
|
||||
|
||||
checkedIterables = self.z.checkbox(name, defaultCheckedIterables, optionIterables)
|
||||
return gateway.jvm.scala.collection.JavaConversions.asJavaCollection(checkedIterables)
|
||||
checkedItems = gateway.jvm.scala.collection.JavaConversions.seqAsJavaList(self.z.checkbox(name, defaultCheckedIterables, optionIterables))
|
||||
result = []
|
||||
for checkedItem in checkedItems:
|
||||
result.append(checkedItem)
|
||||
return result;
|
||||
|
||||
def registerHook(self, event, cmd, replName=None):
|
||||
if replName is None:
|
||||
|
|
@ -271,19 +273,37 @@ else:
|
|||
|
||||
java_import(gateway.jvm, "scala.Tuple2")
|
||||
|
||||
_zcUserQueryNameSpace = {}
|
||||
|
||||
jconf = intp.getSparkConf()
|
||||
conf = SparkConf(_jvm = gateway.jvm, _jconf = jconf)
|
||||
sc = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
|
||||
if sparkVersion.isSpark2():
|
||||
spark = SparkSession(sc, intp.getSparkSession())
|
||||
sqlc = spark._wrapped
|
||||
else:
|
||||
sqlc = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext())
|
||||
sqlContext = sqlc
|
||||
sc = _zsc_ = SparkContext(jsc=jsc, gateway=gateway, conf=conf)
|
||||
_zcUserQueryNameSpace["_zsc_"] = _zsc_
|
||||
_zcUserQueryNameSpace["sc"] = sc
|
||||
|
||||
completion = PySparkCompletion(intp)
|
||||
z = PyZeppelinContext(intp.getZeppelinContext())
|
||||
z._setup_matplotlib()
|
||||
if sparkVersion.isSpark2():
|
||||
spark = __zSpark__ = SparkSession(sc, intp.getSparkSession())
|
||||
sqlc = __zSqlc__ = __zSpark__._wrapped
|
||||
_zcUserQueryNameSpace["sqlc"] = sqlc
|
||||
_zcUserQueryNameSpace["__zSqlc__"] = __zSqlc__
|
||||
_zcUserQueryNameSpace["spark"] = spark
|
||||
_zcUserQueryNameSpace["__zSpark__"] = __zSpark__
|
||||
else:
|
||||
sqlc = __zSqlc__ = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext())
|
||||
_zcUserQueryNameSpace["sqlc"] = sqlc
|
||||
_zcUserQueryNameSpace["__zSqlc__"] = sqlc
|
||||
|
||||
sqlContext = __zSqlc__
|
||||
_zcUserQueryNameSpace["sqlContext"] = sqlContext
|
||||
|
||||
completion = __zeppelin_completion__ = PySparkCompletion(intp)
|
||||
_zcUserQueryNameSpace["completion"] = completion
|
||||
_zcUserQueryNameSpace["__zeppelin_completion__"] = __zeppelin_completion__
|
||||
|
||||
z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext())
|
||||
__zeppelin__._setup_matplotlib()
|
||||
_zcUserQueryNameSpace["z"] = z
|
||||
_zcUserQueryNameSpace["__zeppelin__"] = __zeppelin__
|
||||
|
||||
while True :
|
||||
req = intp.getStatements()
|
||||
|
|
@ -299,7 +319,7 @@ while True :
|
|||
global_hook = None
|
||||
|
||||
try:
|
||||
user_hook = z.getHook('post_exec')
|
||||
user_hook = __zeppelin__.getHook('post_exec')
|
||||
except:
|
||||
user_hook = None
|
||||
|
||||
|
|
@ -334,17 +354,17 @@ while True :
|
|||
for node in to_run_exec:
|
||||
mod = ast.Module([node])
|
||||
code = compile(mod, '<stdin>', 'exec')
|
||||
exec(code)
|
||||
exec(code, _zcUserQueryNameSpace)
|
||||
|
||||
for node in to_run_single:
|
||||
mod = ast.Interactive([node])
|
||||
code = compile(mod, '<stdin>', 'single')
|
||||
exec(code)
|
||||
exec(code, _zcUserQueryNameSpace)
|
||||
|
||||
for node in to_run_hooks:
|
||||
mod = ast.Module([node])
|
||||
code = compile(mod, '<stdin>', 'exec')
|
||||
exec(code)
|
||||
exec(code, _zcUserQueryNameSpace)
|
||||
except:
|
||||
raise Exception(traceback.format_exc())
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,12 @@
|
|||
"propertyName": "spark.master",
|
||||
"defaultValue": "local[*]",
|
||||
"description": "Spark master uri. ex) spark://masterhost:7077"
|
||||
},
|
||||
"zeppelin.spark.unSupportedVersionCheck": {
|
||||
"envName": null,
|
||||
"propertyName": "zeppelin.spark.enableSupportedVersionCheck",
|
||||
"defaultValue": "true",
|
||||
"description": "Do not change - developer only setting, not for production use"
|
||||
}
|
||||
},
|
||||
"editor": {
|
||||
|
|
|
|||
|
|
@ -118,11 +118,26 @@ public class PySparkInterpreterTest {
|
|||
@Test
|
||||
public void testCompletion() {
|
||||
if (getSparkVersionNumber() > 11) {
|
||||
List<InterpreterCompletion> completions = pySparkInterpreter.completion("sc.", "sc.".length());
|
||||
List<InterpreterCompletion> completions = pySparkInterpreter.completion("sc.", "sc.".length(), null);
|
||||
assertTrue(completions.size() > 0);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRedefinitionZeppelinContext() {
|
||||
if (getSparkVersionNumber() > 11) {
|
||||
String redefinitionCode = "z = 1\n";
|
||||
String restoreCode = "z = __zeppelin__\n";
|
||||
String validCode = "z.input(\"test\")\n";
|
||||
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(validCode, context).code());
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(redefinitionCode, context).code());
|
||||
assertEquals(InterpreterResult.Code.ERROR, pySparkInterpreter.interpret(validCode, context).code());
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(restoreCode, context).code());
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, pySparkInterpreter.interpret(validCode, context).code());
|
||||
}
|
||||
}
|
||||
|
||||
private class infinityPythonJob implements Runnable {
|
||||
@Override
|
||||
public void run() {
|
||||
|
|
|
|||
|
|
@ -301,7 +301,7 @@ public class SparkInterpreterTest {
|
|||
|
||||
@Test
|
||||
public void testCompletion() {
|
||||
List<InterpreterCompletion> completions = repl.completion("sc.", "sc.".length());
|
||||
List<InterpreterCompletion> completions = repl.completion("sc.", "sc.".length(), null);
|
||||
assertTrue(completions.size() > 0);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ public class SparkSqlInterpreterTest {
|
|||
public static void setUp() throws Exception {
|
||||
Properties p = new Properties();
|
||||
p.putAll(SparkInterpreterTest.getSparkTestProperties(tmpDir));
|
||||
p.setProperty("zeppelin.spark.maxResult", "1000");
|
||||
p.setProperty("zeppelin.spark.maxResult", "10");
|
||||
p.setProperty("zeppelin.spark.concurrentSQL", "false");
|
||||
p.setProperty("zeppelin.spark.sql.stacktrace", "false");
|
||||
|
||||
|
|
@ -160,4 +160,21 @@ public class SparkSqlInterpreterTest {
|
|||
assertEquals(Type.TABLE, ret.message().get(0).getType());
|
||||
assertEquals("name\tage\ngates\tnull\n", ret.message().get(0).getData());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMaxResults() {
|
||||
repl.interpret("case class P(age:Int)", context);
|
||||
repl.interpret(
|
||||
"val gr = sc.parallelize(Seq(P(1),P(2),P(3),P(4),P(5),P(6),P(7),P(8),P(9),P(10),P(11)))",
|
||||
context);
|
||||
if (isDataFrameSupported()) {
|
||||
repl.interpret("gr.toDF.registerTempTable(\"gr\")", context);
|
||||
} else {
|
||||
repl.interpret("gr.registerTempTable(\"gr\")", context);
|
||||
}
|
||||
|
||||
InterpreterResult ret = sql.interpret("select * from gr", context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
|
||||
assertTrue(ret.message().get(1).getData().contains("alert-warning"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@
|
|||
set -ev
|
||||
touch ~/.environ
|
||||
|
||||
# Install R dependencies if R profiles are used
|
||||
if [[ ${PROFILE/"-Pr "} != $PROFILE ]] || [[ ${PROFILE/"-Psparkr "} != $PROFILE ]] ; then
|
||||
# Install R dependencies if SPARKR is true
|
||||
if [[ "${SPARKR}" = "true" ]] ; then
|
||||
echo "R_LIBS=~/R" > ~/.Renviron
|
||||
echo "export R_LIBS=~/R" >> ~/.environ
|
||||
source ~/.environ
|
||||
|
|
|
|||
|
|
@ -229,7 +229,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
|
|||
(The MIT License) Json3 v3.3.1 (http://bestiejs.github.io/json3) - https://github.com/bestiejs/json3/blob/v3.3.1/LICENSE
|
||||
(The MIT License) es5-shim v3.1.0 (https://github.com/es-shims/es5-shim) - https://github.com/es-shims/es5-shim/blob/v3.1.0/LICENSE
|
||||
(The MIT License) bootstrap v3.2.0 (http://getbootstrap.com/) - https://github.com/twbs/bootstrap/blob/v3.2.0/LICENSE
|
||||
(The MIT License) UI Bootstrap v0.13.0 (http://angular-ui.github.io/bootstrap/) - https://github.com/angular-ui/bootstrap/blob/0.13.0/LICENSE
|
||||
(The MIT License) UI Bootstrap v2.5.0 (http://angular-ui.github.io/bootstrap/) - https://github.com/angular-ui/bootstrap/blob/2.5.0/LICENSE
|
||||
(The MIT License) bootstrap3-dialog v1.34.7 (https://github.com/nakupanda/bootstrap3-dialog/tree/v1.34.7) - https://github.com/nakupanda/bootstrap3-dialog/tree/v1.34.7
|
||||
(The MIT License) Angular Websocket v1.0.13 (http://angularclass.github.io/angular-websocket/) - https://github.com/AngularClass/angular-websocket/blob/v1.0.13/LICENSE
|
||||
(The MIT License) UI.Ace v0.1.1 (http://angularclass.github.io/angular-websocket/) - https://github.com/angular-ui/ui-ace/blob/master/LICENSE
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@
|
|||
<aether.version>1.12</aether.version>
|
||||
<maven.aeither.provider.version>3.0.3</maven.aeither.provider.version>
|
||||
<wagon.version>1.0</wagon.version>
|
||||
<jline.version>2.12.1</jline.version>
|
||||
|
||||
<!--plugin versions-->
|
||||
<plugin.shade.version>2.3</plugin.shade.version>
|
||||
|
|
@ -60,6 +61,11 @@
|
|||
<artifactId>gson</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.danilopianini</groupId>
|
||||
<artifactId>gson-extras</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-exec</artifactId>
|
||||
|
|
@ -202,6 +208,17 @@
|
|||
<version>${wagon.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>jline</groupId>
|
||||
<artifactId>jline</artifactId>
|
||||
<version>${jline.version}</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
|
||||
* agreements. See the NOTICE file distributed with this work for additional information regarding
|
||||
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License. You may obtain a
|
||||
* copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied. See the License for the specific language governing permissions and limitations under
|
||||
* the License.
|
||||
*/
|
||||
package org.apache.zeppelin.completer;
|
||||
|
||||
/**
|
||||
* Types of completion
|
||||
*/
|
||||
public enum CompletionType {
|
||||
schema,
|
||||
table,
|
||||
column,
|
||||
setting,
|
||||
command,
|
||||
keyword,
|
||||
path
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
|
||||
* agreements. See the NOTICE file distributed with this work for additional information regarding
|
||||
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License. You may obtain a
|
||||
* copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied. See the License for the specific language governing permissions and limitations under
|
||||
* the License.
|
||||
*/
|
||||
package org.apache.zeppelin.completer;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import jline.console.completer.Completer;
|
||||
import jline.internal.Preconditions;
|
||||
|
||||
/**
|
||||
* Case-insensitive completer for a set of strings.
|
||||
*/
|
||||
public class StringsCompleter implements Completer {
|
||||
private final SortedSet<String> strings = new TreeSet<String>(new Comparator<String>() {
|
||||
@Override
|
||||
public int compare(String o1, String o2) {
|
||||
return o1.compareToIgnoreCase(o2);
|
||||
}
|
||||
});
|
||||
|
||||
public StringsCompleter() {
|
||||
}
|
||||
|
||||
public StringsCompleter(final Collection<String> strings) {
|
||||
Preconditions.checkNotNull(strings);
|
||||
getStrings().addAll(strings);
|
||||
}
|
||||
|
||||
public Collection<String> getStrings() {
|
||||
return strings;
|
||||
}
|
||||
|
||||
public int complete(final String buffer, final int cursor, final List<CharSequence> candidates) {
|
||||
return completeCollection(buffer, cursor, candidates);
|
||||
}
|
||||
|
||||
public int complete(final String buffer, final int cursor, final Set<CharSequence> candidates) {
|
||||
return completeCollection(buffer, cursor, candidates);
|
||||
}
|
||||
|
||||
private int completeCollection(final String buffer, final int cursor,
|
||||
final Collection<CharSequence> candidates) {
|
||||
Preconditions.checkNotNull(candidates);
|
||||
if (buffer == null) {
|
||||
candidates.addAll(strings);
|
||||
} else {
|
||||
String bufferTmp = buffer.toUpperCase();
|
||||
for (String match : strings.tailSet(buffer)) {
|
||||
String matchTmp = match.toUpperCase();
|
||||
if (!matchTmp.startsWith(bufferTmp)) {
|
||||
break;
|
||||
}
|
||||
|
||||
candidates.add(match);
|
||||
}
|
||||
}
|
||||
|
||||
return candidates.isEmpty() ? -1 : 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -34,6 +34,8 @@ import org.slf4j.LoggerFactory;
|
|||
* @param <T>
|
||||
*/
|
||||
public class AngularObject<T> {
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(AngularObject.class);
|
||||
|
||||
private String name;
|
||||
private T object;
|
||||
|
||||
|
|
@ -172,7 +174,7 @@ public class AngularObject<T> {
|
|||
if (emit) {
|
||||
emit();
|
||||
}
|
||||
|
||||
LOGGER.debug("Update angular object: " + name + " with value: " + o);
|
||||
final Logger logger = LoggerFactory.getLogger(AngularObject.class);
|
||||
List<AngularObjectWatcher> ws = new LinkedList<>();
|
||||
synchronized (watchers) {
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue