mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Merge branch 'master' into ZEPPELIN-732-up
This commit is contained in:
Lee moon soo
commit
c714a1950c
38 changed files with 923 additions and 578 deletions
45
bin/install-interpreter.sh
Executable file
45
bin/install-interpreter.sh
Executable file
|
|
@ -0,0 +1,45 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# Run Zeppelin
|
||||
#
|
||||
|
||||
bin=$(dirname "${BASH_SOURCE-$0}")
|
||||
bin=$(cd "${bin}">/dev/null; pwd)
|
||||
|
||||
. "${bin}/common.sh"
|
||||
|
||||
|
||||
ZEPPELIN_INSTALL_INTERPRETER_MAIN=org.apache.zeppelin.interpreter.install.InstallInterpreter
|
||||
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/install-interpreter.log"
|
||||
JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}"
|
||||
|
||||
if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
|
||||
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
|
||||
fi
|
||||
addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
|
||||
|
||||
if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
|
||||
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
|
||||
fi
|
||||
addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
|
||||
|
||||
addJarInDir "${ZEPPELIN_HOME}/lib"
|
||||
|
||||
CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
|
||||
$ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_INSTALL_INTERPRETER_MAIN ${@}
|
||||
35
conf/interpreter-list
Normal file
35
conf/interpreter-list
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
#
|
||||
# [name] [maven artifact] [description]
|
||||
|
||||
alluxio org.apache.zeppelin:zeppelin-alluxio:0.6.0 Alluxio interpreter
|
||||
angular org.apache.zeppelin:zeppelin-angular:0.6.0 HTML and AngularJS view rendering
|
||||
cassandra org.apache.zeppelin:zeppelin-cassandra:0.6.0 Cassandra interpreter
|
||||
elasticsearch org.apache.zeppelin:zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter
|
||||
file org.apache.zeppelin:zeppelin-file:0.6.0 HDFS file interpreter
|
||||
flink org.apache.zeppelin:zeppelin-flink:0.6.0 Flink interpreter
|
||||
hbase org.apache.zeppelin:zeppelin-hbase:0.6.0 Hbase interpreter
|
||||
ignite org.apache.zeppelin:zeppelin-ignite:0.6.0 Ignite interpreter
|
||||
jdbc org.apache.zeppelin:zeppelin-jdbc:0.6.0 Jdbc interpreter
|
||||
kylin org.apache.zeppelin:zeppelin-kylin:0.6.0 Kylin interpreter
|
||||
lens org.apache.zeppelin:zeppelin-lens:0.6.0 Lens interpreter
|
||||
livy org.apache.zeppelin:zeppelin-livy:0.6.0 Livy interpreter
|
||||
md org.apache.zeppelin:zeppelin-markdown:0.6.0 Markdown support
|
||||
postgresql org.apache.zeppelin:zeppelin-postgresql:0.6.0 Postgresql interpreter
|
||||
python org.apache.zeppelin:zeppelin-python:0.6.0 Python interpreter
|
||||
shell org.apache.zeppelin:zeppelin-shell:0.6.0 Shell command
|
||||
|
|
@ -55,12 +55,13 @@ REM set HADOOP_CONF_DIR REM yarn-site.xml is located in configuration
|
|||
REM Pyspark (supported with Spark 1.2.1 and above)
|
||||
REM To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI
|
||||
REM set PYSPARK_PYTHON REM path to the python command. must be the same path on the driver(Zeppelin) and all workers.
|
||||
REM set PYTHONPATH
|
||||
REM set PYTHONPATH
|
||||
|
||||
REM Spark interpreter options
|
||||
REM
|
||||
REM set ZEPPELIN_SPARK_USEHIVECONTEXT REM Use HiveContext instead of SQLContext if set true. true by default.
|
||||
REM set ZEPPELIN_SPARK_CONCURRENTSQL REM Execute multiple SQL concurrently if set true. false by default.
|
||||
REM set ZEPPELIN_SPARK_IMPORTIMPLICIT REM Import implicits, UDF collection, and sql if set true. true by default.
|
||||
REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of SparkSQL result to display. 1000 by default.
|
||||
|
||||
REM ZeppelinHub connection configuration
|
||||
|
|
|
|||
|
|
@ -55,15 +55,17 @@
|
|||
# Pyspark (supported with Spark 1.2.1 and above)
|
||||
# To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI
|
||||
# export PYSPARK_PYTHON # path to the python command. must be the same path on the driver(Zeppelin) and all workers.
|
||||
# export PYTHONPATH
|
||||
# export PYTHONPATH
|
||||
|
||||
## Spark interpreter options ##
|
||||
##
|
||||
# export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default.
|
||||
# export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default.
|
||||
# export ZEPPELIN_SPARK_IMPORTIMPLICIT # Import implicits, UDF collection, and sql if set true. true by default.
|
||||
# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default.
|
||||
# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000
|
||||
|
||||
|
||||
#### HBase interpreter configuration ####
|
||||
|
||||
## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set
|
||||
|
|
@ -75,4 +77,3 @@
|
|||
# export ZEPPELINHUB_API_ADDRESS # Refers to the address of the ZeppelinHub service in use
|
||||
# export ZEPPELINHUB_API_TOKEN # Refers to the Zeppelin instance token of the user
|
||||
# export ZEPPELINHUB_USER_KEY # Optional, when using Zeppelin with authentication.
|
||||
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ function make_binary_release() {
|
|||
git_clone
|
||||
make_source_package
|
||||
make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr"
|
||||
make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -pl !alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell"
|
||||
|
||||
# remove non release files and dirs
|
||||
rm -rf "${WORKING_DIR}/zeppelin"
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@
|
|||
<li><a href="{{BASE_PATH}}/index.html">What is Apache Zeppelin ?</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li class="title"><span><b>Getting Started</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/install/install.html">Quick Start</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/install.html#zeppelin-configuration">Configuration</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/install.html">Install</a></li>
|
||||
<li><a href="{{BASE_PATH}}/install/install.html#apache-zeppelin-configuration">Configuration</a></li>
|
||||
<li><a href="{{BASE_PATH}}/quickstart/explorezeppelinui.html">Explore Zeppelin UI</a></li>
|
||||
<li><a href="{{BASE_PATH}}/quickstart/tutorial.html">Tutorial</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
|
|
@ -42,6 +42,7 @@
|
|||
<li><a href="{{BASE_PATH}}/manual/interpreters.html">Overview</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
<li class="title"><span><b>Usage</b><span></li>
|
||||
<li><a href="{{BASE_PATH}}/manual/interpreterinstallation.html">Interpreter Installation</a></li>
|
||||
<li><a href="{{BASE_PATH}}/manual/dynamicinterpreterload.html">Dynamic Interpreter Loading</a></li>
|
||||
<li><a href="{{BASE_PATH}}/manual/dependencymanagement.html">Interpreter Dependency Management</a></li>
|
||||
<li role="separator" class="divider"></li>
|
||||
|
|
@ -111,3 +112,4 @@
|
|||
</nav><!--/.navbar-collapse -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -60,6 +60,9 @@ Although it can be unstable somehow since it is on development status, you can e
|
|||
### Downloading Binary Package
|
||||
|
||||
If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html).
|
||||
|
||||
If you have downloaded `netinst` binary, [install additional interpreters](../manual/interpreterinstallation.html) before you start Zeppelin. Or simply run `./bin/install-interpreter.sh --all`.
|
||||
|
||||
After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section.
|
||||
|
||||
### Building from Source
|
||||
|
|
@ -388,4 +391,4 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z
|
|||
<td>1024000</td>
|
||||
<td>Size in characters of the maximum text message to be received by websocket.</td>
|
||||
</tr>
|
||||
</table>
|
||||
</table>
|
||||
|
|
@ -83,7 +83,7 @@ plt.figure()
|
|||
z.show(plt)
|
||||
plt.close()
|
||||
```
|
||||
zeppelin_show function can take optional parameters to adapt graph width and height
|
||||
z.show function can take optional parameters to adapt graph width and height
|
||||
|
||||
```python
|
||||
%python
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ Spark Interpreter group, which consists of five interpreters.
|
|||
</table>
|
||||
|
||||
## Configuration
|
||||
The Spark interpreter can be configured with properties provided by Zeppelin.
|
||||
The Spark interpreter can be configured with properties provided by Zeppelin.
|
||||
You can also set other Spark properties which are not listed in the table. For a list of additional properties, refer to [Spark Available Properties](http://spark.apache.org/docs/latest/configuration.html#available-properties).
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
|
|
@ -111,6 +111,11 @@ You can also set other Spark properties which are not listed in the table. For a
|
|||
<td>true</td>
|
||||
<td>Use HiveContext instead of SQLContext if it is true.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>zeppelin.spark.importImplicit</td>
|
||||
<td>true</td>
|
||||
<td>Import implicits, UDF collection, and sql if set true.</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.
|
||||
|
|
|
|||
164
docs/manual/interpreterinstallation.md
Normal file
164
docs/manual/interpreterinstallation.md
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
---
|
||||
layout: page
|
||||
title: "Interpreter Installation"
|
||||
description: ""
|
||||
group: manual
|
||||
---
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
{% include JB/setup %}
|
||||
|
||||
# Interpreter Installation
|
||||
|
||||
Apache Zeppelin provides **Interpreter Installation** mechanism for whom downloaded Zeppelin `netinst` binary package, or just want to install another 3rd party interpreters.
|
||||
|
||||
## Community managed interpreters
|
||||
Apache Zeppelin provides several interpreters as [community managed interpreters](#available-community-managed-interpreters).
|
||||
If you downloaded `netinst` binary package, you need to install by using below commands.
|
||||
|
||||
#### Install all community managed interpreters
|
||||
|
||||
```
|
||||
./bin/install-interpreter.sh --all
|
||||
```
|
||||
|
||||
#### Install specific interpreters
|
||||
|
||||
```
|
||||
./bin/install-interpreter.sh --name md,shell,jdbc,python
|
||||
```
|
||||
|
||||
You can get full list of community managed interpreters by running
|
||||
|
||||
```
|
||||
./bin/install-interpreter.sh --list
|
||||
```
|
||||
|
||||
Once you have installed interpreters, you need to restart Zeppelin. And then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
|
||||
|
||||
|
||||
## 3rd party interpreters
|
||||
|
||||
You can also install 3rd party interpreters located in the maven repository by using below commands.
|
||||
|
||||
#### Install 3rd party interpreters
|
||||
|
||||
```
|
||||
./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1
|
||||
```
|
||||
|
||||
The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory.
|
||||
|
||||
Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#apache-zeppelin-configuration).
|
||||
And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
|
||||
|
||||
|
||||
#### Install multiple 3rd party interpreters at once
|
||||
|
||||
```
|
||||
./bin/install-interpreter.sh --name interpreter1,interpreter2 --artifact groupId1:artifact1:version1,groupId2:artifact2:version2
|
||||
```
|
||||
|
||||
`--name` and `--artifact` arguments will recieve comma separated list.
|
||||
|
||||
## Available community managed interpreters
|
||||
|
||||
You can also find the below community managed interpreter list in `conf/interpreter-list` file.
|
||||
<table class="table-configuration">
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Maven Artifact</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>alluxio</td>
|
||||
<td>org.apache.zeppelin:zeppelin-alluxio:0.6.0</td>
|
||||
<td>Alluxio interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>angular</td>
|
||||
<td>org.apache.zeppelin:zeppelin-angular:0.6.0</td>
|
||||
<td>HTML and AngularJS view rendering</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>cassandra</td>
|
||||
<td>org.apache.zeppelin:zeppelin-cassandra:0.6.0</td>
|
||||
<td>Cassandra interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>elasticsearch</td>
|
||||
<td>org.apache.zeppelin:zeppelin-elasticsearch:0.6.0</td>
|
||||
<td>Elasticsearch interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>file</td>
|
||||
<td>org.apache.zeppelin:zeppelin-file:0.6.0</td>
|
||||
<td>HDFS file interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>flink</td>
|
||||
<td>org.apache.zeppelin:zeppelin-flink:0.6.0</td>
|
||||
<td>Flink interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>hbase</td>
|
||||
<td>org.apache.zeppelin:zeppelin-hbase:0.6.0</td>
|
||||
<td>Hbase interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ignite</td>
|
||||
<td>org.apache.zeppelin:zeppelin-ignite:0.6.0</td>
|
||||
<td>Ignite interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>jdbc</td>
|
||||
<td>org.apache.zeppelin:zeppelin-jdbc:0.6.0</td>
|
||||
<td>Jdbc interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>kylin</td>
|
||||
<td>org.apache.zeppelin:zeppelin-kylin:0.6.0</td>
|
||||
<td>Kylin interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>lens</td>
|
||||
<td>org.apache.zeppelin:zeppelin-lens:0.6.0</td>
|
||||
<td>Lens interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>livy</td>
|
||||
<td>org.apache.zeppelin:zeppelin-livy:0.6.0</td>
|
||||
<td>Livy interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>md</td>
|
||||
<td>org.apache.zeppelin:zeppelin-markdown:0.6.0</td>
|
||||
<td>Markdown support</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>postgresql</td>
|
||||
<td>org.apache.zeppelin:zeppelin-postgresql:0.6.0</td>
|
||||
<td>Postgresql interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>python</td>
|
||||
<td>org.apache.zeppelin:zeppelin-python:0.6.0</td>
|
||||
<td>Python interpreter</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>shell</td>
|
||||
<td>org.apache.zeppelin:zeppelin-shell:0.6.0</td>
|
||||
<td>Shell command</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
|
@ -116,7 +116,7 @@ public class PythonInterpreterTest {
|
|||
assertTrue(cmdHistory.contains("def help()"));
|
||||
assertTrue(cmdHistory.contains("class PyZeppelinContext(object):"));
|
||||
assertTrue(cmdHistory.contains("z = PyZeppelinContext"));
|
||||
assertTrue(cmdHistory.contains("z.show"));
|
||||
assertTrue(cmdHistory.contains("def show"));
|
||||
assertFalse(cmdHistory.contains("GatewayClient"));
|
||||
|
||||
}
|
||||
|
|
@ -141,8 +141,8 @@ public class PythonInterpreterTest {
|
|||
|
||||
assertTrue(cmdHistory.contains("def help()"));
|
||||
assertTrue(cmdHistory.contains("class PyZeppelinContext(object):"));
|
||||
assertTrue(cmdHistory.contains("z = PyZeppelinContext"));
|
||||
assertTrue(cmdHistory.contains("z.show"));
|
||||
assertTrue(cmdHistory.contains("z = Py4jZeppelinContext"));
|
||||
assertTrue(cmdHistory.contains("def show"));
|
||||
assertTrue(cmdHistory.contains("GatewayClient(port=" + py4jPort + ")"));
|
||||
assertTrue(cmdHistory.contains("org.apache.zeppelin.display.Input"));
|
||||
|
||||
|
|
|
|||
|
|
@ -178,6 +178,10 @@ public class SparkInterpreter extends Interpreter {
|
|||
return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.useHiveContext"));
|
||||
}
|
||||
|
||||
private boolean importImplicit() {
|
||||
return java.lang.Boolean.parseBoolean(getProperty("zeppelin.spark.importImplicit"));
|
||||
}
|
||||
|
||||
public SQLContext getSQLContext() {
|
||||
synchronized (sharedInterpreterLock) {
|
||||
if (sqlc == null) {
|
||||
|
|
@ -252,7 +256,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
| IllegalArgumentException | InvocationTargetException e) {
|
||||
// continue instead of: throw new InterpreterException(e);
|
||||
// Newer Spark versions (like the patched CDH5.7.0 one) don't contain this method
|
||||
logger.warn(String.format("Spark method classServerUri not available due to: [%s]",
|
||||
logger.warn(String.format("Spark method classServerUri not available due to: [%s]",
|
||||
e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
|
@ -544,12 +548,14 @@ public class SparkInterpreter extends Interpreter {
|
|||
+ "_binder.get(\"sqlc\").asInstanceOf[org.apache.spark.sql.SQLContext]");
|
||||
intp.interpret("import org.apache.spark.SparkContext._");
|
||||
|
||||
if (sparkVersion.oldSqlContextImplicits()) {
|
||||
intp.interpret("import sqlContext._");
|
||||
} else {
|
||||
intp.interpret("import sqlContext.implicits._");
|
||||
intp.interpret("import sqlContext.sql");
|
||||
intp.interpret("import org.apache.spark.sql.functions._");
|
||||
if (importImplicit()) {
|
||||
if (sparkVersion.oldSqlContextImplicits()) {
|
||||
intp.interpret("import sqlContext._");
|
||||
} else {
|
||||
intp.interpret("import sqlContext.implicits._");
|
||||
intp.interpret("import sqlContext.sql");
|
||||
intp.interpret("import org.apache.spark.sql.functions._");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,12 @@
|
|||
"propertyName": "zeppelin.spark.maxResult",
|
||||
"defaultValue": "1000",
|
||||
"description": "Max number of SparkSQL result to display."
|
||||
},
|
||||
"zeppelin.spark.importImplicit": {
|
||||
"envName": "ZEPPELIN_SPARK_IMPORTIMPLICIT",
|
||||
"propertyName": "zeppelin.spark.importImplicit",
|
||||
"defaultValue": "true",
|
||||
"description": "Import implicits, UDF collection, and sql if set true. true by default."
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ public class SparkInterpreterTest {
|
|||
p.setProperty("spark.app.name", "Zeppelin Test");
|
||||
p.setProperty("zeppelin.spark.useHiveContext", "true");
|
||||
p.setProperty("zeppelin.spark.maxResult", "1000");
|
||||
p.setProperty("zeppelin.spark.importImplicit", "true");
|
||||
|
||||
return p;
|
||||
}
|
||||
|
|
@ -229,4 +230,35 @@ public class SparkInterpreterTest {
|
|||
|
||||
repl2.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEnableImplicitImport() {
|
||||
// Set option of importing implicits to "true", and initialize new Spark repl
|
||||
Properties p = getSparkTestProperties();
|
||||
p.setProperty("zeppelin.spark.importImplicit", "true");
|
||||
SparkInterpreter repl2 = new SparkInterpreter(p);
|
||||
repl2.setInterpreterGroup(intpGroup);
|
||||
intpGroup.get("note").add(repl2);
|
||||
|
||||
repl2.open();
|
||||
String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
|
||||
assertEquals(Code.SUCCESS, repl2.interpret(ddl, context).code());
|
||||
repl2.close();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDisableImplicitImport() {
|
||||
// Set option of importing implicits to "false", and initialize new Spark repl
|
||||
// this test should return error status when creating DataFrame from sequence
|
||||
Properties p = getSparkTestProperties();
|
||||
p.setProperty("zeppelin.spark.importImplicit", "false");
|
||||
SparkInterpreter repl2 = new SparkInterpreter(p);
|
||||
repl2.setInterpreterGroup(intpGroup);
|
||||
intpGroup.get("note").add(repl2);
|
||||
|
||||
repl2.open();
|
||||
String ddl = "val df = Seq((1, true), (2, false)).toDF(\"num\", \"bool\")";
|
||||
assertEquals(Code.ERROR, repl2.interpret(ddl, context).code());
|
||||
repl2.close();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.zeppelin.dep;
|
||||
|
||||
import java.net.URL;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
|
|
@ -25,6 +26,7 @@ import java.util.List;
|
|||
import org.sonatype.aether.RepositorySystem;
|
||||
import org.sonatype.aether.RepositorySystemSession;
|
||||
import org.sonatype.aether.repository.Authentication;
|
||||
import org.sonatype.aether.repository.Proxy;
|
||||
import org.sonatype.aether.repository.RemoteRepository;
|
||||
import org.sonatype.aether.repository.RepositoryPolicy;
|
||||
import org.sonatype.aether.resolution.ArtifactResult;
|
||||
|
|
@ -44,6 +46,16 @@ public abstract class AbstractDependencyResolver {
|
|||
repos.add(Booter.newLocalRepository());
|
||||
}
|
||||
|
||||
public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) {
|
||||
Authentication auth = new Authentication(proxyUser, proxyPassword);
|
||||
Proxy proxy = new Proxy(proxyUrl.getProtocol(), proxyUrl.getHost(), proxyUrl.getPort(), auth);
|
||||
synchronized (repos) {
|
||||
for (RemoteRepository repo : repos) {
|
||||
repo.setProxy(proxy);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public List<RemoteRepository> getRepos() {
|
||||
return this.repos;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.zeppelin.dep;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
|
|
@ -63,11 +64,6 @@ public class DependencyResolver extends AbstractDependencyResolver {
|
|||
return load(artifact, new LinkedList<String>());
|
||||
}
|
||||
|
||||
public List<File> load(String artifact, String destPath)
|
||||
throws RepositoryException, IOException {
|
||||
return load(artifact, new LinkedList<String>(), destPath);
|
||||
}
|
||||
|
||||
public synchronized List<File> load(String artifact, Collection<String> excludes)
|
||||
throws RepositoryException, IOException {
|
||||
if (StringUtils.isBlank(artifact)) {
|
||||
|
|
@ -86,24 +82,11 @@ public class DependencyResolver extends AbstractDependencyResolver {
|
|||
}
|
||||
}
|
||||
|
||||
private String getPath(String path) {
|
||||
if (path.startsWith("/")) {
|
||||
return path;
|
||||
} else {
|
||||
// find home dir
|
||||
String home = System.getenv("ZEPPELIN_HOME");
|
||||
if (home == null) {
|
||||
home = System.getProperty("zeppelin.home");
|
||||
}
|
||||
if (home == null) {
|
||||
home = "..";
|
||||
}
|
||||
|
||||
return home + "/" + path;
|
||||
}
|
||||
public List<File> load(String artifact, File destPath) throws IOException, RepositoryException {
|
||||
return load(artifact, new LinkedList<String>(), destPath);
|
||||
}
|
||||
|
||||
public List<File> load(String artifact, Collection<String> excludes, String destPath)
|
||||
public List<File> load(String artifact, Collection<String> excludes, File destPath)
|
||||
throws RepositoryException, IOException {
|
||||
List<File> libs = new LinkedList<File>();
|
||||
|
||||
|
|
@ -111,7 +94,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
|
|||
libs = load(artifact, excludes);
|
||||
|
||||
for (File srcFile : libs) {
|
||||
File destFile = new File(getPath(destPath), srcFile.getName());
|
||||
File destFile = new File(destPath, srcFile.getName());
|
||||
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
|
||||
FileUtils.copyFile(srcFile, destFile);
|
||||
logger.info("copy {} to {}", srcFile.getAbsolutePath(), getPath(destPath));
|
||||
|
|
|
|||
|
|
@ -33,27 +33,20 @@ import org.sonatype.aether.RepositoryException;
|
|||
public class DependencyResolverTest {
|
||||
private static DependencyResolver resolver;
|
||||
private static String testPath;
|
||||
private static String testCopyPath;
|
||||
private static String home;
|
||||
|
||||
private static File testCopyPath;
|
||||
private static File tmpDir;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
testPath = "test-repo";
|
||||
testCopyPath = "test-copy-repo";
|
||||
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
|
||||
testPath = tmpDir.getAbsolutePath() + "/test-repo";
|
||||
testCopyPath = new File(tmpDir, "test-copy-repo");
|
||||
resolver = new DependencyResolver(testPath);
|
||||
home = System.getenv("ZEPPELIN_HOME");
|
||||
if (home == null) {
|
||||
home = System.getProperty("zeppelin.home");
|
||||
}
|
||||
if (home == null) {
|
||||
home = "..";
|
||||
}
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() throws Exception {
|
||||
FileUtils.deleteDirectory(new File(home + "/" + testPath));
|
||||
FileUtils.deleteDirectory(new File(home + "/" + testCopyPath));
|
||||
FileUtils.deleteDirectory(tmpDir);
|
||||
}
|
||||
|
||||
@Rule
|
||||
|
|
@ -78,19 +71,19 @@ public class DependencyResolverTest {
|
|||
public void testLoad() throws Exception {
|
||||
// basic load
|
||||
resolver.load("com.databricks:spark-csv_2.10:1.3.0", testCopyPath);
|
||||
assertEquals(new File(home + "/" + testCopyPath).list().length, 4);
|
||||
FileUtils.cleanDirectory(new File(home + "/" + testCopyPath));
|
||||
assertEquals(testCopyPath.list().length, 4);
|
||||
FileUtils.cleanDirectory(testCopyPath);
|
||||
|
||||
// load with exclusions parameter
|
||||
resolver.load("com.databricks:spark-csv_2.10:1.3.0",
|
||||
Collections.singletonList("org.scala-lang:scala-library"), testCopyPath);
|
||||
assertEquals(new File(home + "/" + testCopyPath).list().length, 3);
|
||||
FileUtils.cleanDirectory(new File(home + "/" + testCopyPath));
|
||||
assertEquals(testCopyPath.list().length, 3);
|
||||
FileUtils.cleanDirectory(testCopyPath);
|
||||
|
||||
// load from added repository
|
||||
resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/", false);
|
||||
resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath);
|
||||
assertEquals(new File(home + "/" + testCopyPath).list().length, 8);
|
||||
assertEquals(testCopyPath.list().length, 8);
|
||||
|
||||
// load invalid artifact
|
||||
resolver.delRepo("sonatype");
|
||||
|
|
|
|||
|
|
@ -256,13 +256,6 @@
|
|||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.3.6</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.scalatest</groupId>
|
||||
<artifactId>scalatest_2.10</artifactId>
|
||||
|
|
|
|||
|
|
@ -79,8 +79,7 @@ public class NotebookServer extends WebSocketServlet implements
|
|||
}
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class);
|
||||
Gson gson = new GsonBuilder()
|
||||
.setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
|
||||
Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
|
||||
final Map<String, List<NotebookSocket>> noteSocketMap = new HashMap<>();
|
||||
final Queue<NotebookSocket> connectedSockets = new ConcurrentLinkedQueue<>();
|
||||
|
||||
|
|
|
|||
|
|
@ -125,8 +125,7 @@ public class WebDriverManager {
|
|||
(new WebDriverWait(driver, 5)).until(new ExpectedCondition<Boolean>() {
|
||||
@Override
|
||||
public Boolean apply(WebDriver d) {
|
||||
return d.findElement(By.xpath(
|
||||
"//div[contains(@class, 'navbar-collapse')]//li//a[contains(.,'Connected')]"))
|
||||
return d.findElement(By.xpath("//i[@tooltip='WebSocket Connected']"))
|
||||
.isDisplayed();
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -158,8 +158,8 @@ public class AuthenticationIT extends AbstractZeppelinIT {
|
|||
|
||||
String noteId = driver.getCurrentUrl().substring(driver.getCurrentUrl().lastIndexOf("/") + 1);
|
||||
|
||||
pollingWait(By.xpath("//button[@tooltip='Note permissions']"),
|
||||
MAX_BROWSER_TIMEOUT_SEC).sendKeys(Keys.ENTER);
|
||||
pollingWait(By.xpath("//span[@tooltip='Note permissions']"),
|
||||
MAX_BROWSER_TIMEOUT_SEC).click();
|
||||
pollingWait(By.xpath("//input[@ng-model='permissions.owners']"), MAX_BROWSER_TIMEOUT_SEC)
|
||||
.sendKeys("finance");
|
||||
pollingWait(By.xpath("//input[@ng-model='permissions.readers']"), MAX_BROWSER_TIMEOUT_SEC)
|
||||
|
|
|
|||
|
|
@ -196,7 +196,9 @@ public class ZeppelinIT extends AbstractZeppelinIT {
|
|||
}
|
||||
try {
|
||||
// navigate to interpreter page
|
||||
WebElement interpreterLink = driver.findElement(By.xpath("//a[contains(.,'Interpreter')]"));
|
||||
WebElement settingButton = driver.findElement(By.xpath("//button[@class='nav-btn dropdown-toggle ng-scope']"));
|
||||
settingButton.click();
|
||||
WebElement interpreterLink = driver.findElement(By.xpath("//a[@href='#/interpreter']"));
|
||||
interpreterLink.click();
|
||||
|
||||
// add new dependency to spark interpreter
|
||||
|
|
@ -235,6 +237,7 @@ public class ZeppelinIT extends AbstractZeppelinIT {
|
|||
sleep(1000, false);
|
||||
|
||||
// reset dependency
|
||||
settingButton.click();
|
||||
interpreterLink.click();
|
||||
driver.findElement(By.xpath("//div[@id='spark']//button[contains(.,'edit')]")).sendKeys(Keys.ENTER);
|
||||
WebElement testDepRemoveBtn = pollingWait(By.xpath("//tr[descendant::text()[contains(.,'" +
|
||||
|
|
|
|||
|
|
@ -127,8 +127,9 @@ public abstract class AbstractTestRestApi {
|
|||
}
|
||||
}
|
||||
|
||||
// set spark master
|
||||
// set spark master and other properties
|
||||
sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
|
||||
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
|
||||
|
||||
// set spark home for pyspark
|
||||
sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome());
|
||||
|
|
|
|||
|
|
@ -242,10 +242,6 @@ a.navbar-brand:hover {
|
|||
background: #080808;
|
||||
}
|
||||
|
||||
.server-status {
|
||||
float: right;
|
||||
}
|
||||
|
||||
.navbar-inverse .navbar-nav .open .dropdown-menu .divider {
|
||||
background: #3071A9;
|
||||
}
|
||||
|
|
@ -272,20 +268,43 @@ a.navbar-brand:hover {
|
|||
}
|
||||
}
|
||||
|
||||
i.server-status {
|
||||
font-size: 12px;
|
||||
top: -2px;
|
||||
position: relative;
|
||||
.nav-component {
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.nav-btn {
|
||||
color: #fff;
|
||||
background-color: transparent;
|
||||
border-color: transparent;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.nav-login-btn,
|
||||
.nav-login-btn:hover,
|
||||
.nav-login-btn:focus {
|
||||
color: #fff;
|
||||
background-color: #428bca;
|
||||
border-color: #357ebd;
|
||||
}
|
||||
|
||||
.username {
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
max-width: 120px;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.server-connected {
|
||||
padding-top: 12px;
|
||||
color: #00CC00;
|
||||
font-size: 12px !important;
|
||||
}
|
||||
|
||||
.server-disconnected {
|
||||
padding-top: 12px;
|
||||
color: rgba(240, 48, 0, 1);
|
||||
font-size: 12px !important;
|
||||
font-weight: bold !important;
|
||||
}
|
||||
|
||||
.box {
|
||||
|
|
|
|||
|
|
@ -153,26 +153,24 @@ limitations under the License.
|
|||
</span>
|
||||
|
||||
<div class="pull-right" style="margin-top:15px; margin-right:15px; font-size:15px;">
|
||||
<span>
|
||||
<button type="button"
|
||||
class="btn btn-default btn-xs"
|
||||
<span class="setting-btn"
|
||||
type="button"
|
||||
data-toggle="modal"
|
||||
data-target="#shortcutModal"
|
||||
tooltip-placement="bottom" tooltip="List of shortcut">
|
||||
<i class="fa fa-keyboard-o"></i>
|
||||
</button>
|
||||
<button type="button"
|
||||
class="btn btn-default btn-xs"
|
||||
<i class="fa fa-keyboard-o"></i>
|
||||
</span>
|
||||
<span class="setting-btn"
|
||||
type="button"
|
||||
ng-click="toggleSetting()"
|
||||
tooltip-placement="bottom" tooltip="Interpreter binding">
|
||||
<i class="fa fa-cog" ng-style="{color: showSetting ? '#3071A9' : 'black' }"></i>
|
||||
</button>
|
||||
<button type="button"
|
||||
class="btn btn-default btn-xs"
|
||||
<i class="fa fa-cog" ng-style="{color: showSetting ? '#3071A9' : 'black' }"></i>
|
||||
</span>
|
||||
<span class="setting-btn"
|
||||
type="button"
|
||||
ng-click="togglePermissions()"
|
||||
tooltip-placement="bottom" tooltip="Note permissions">
|
||||
<i class="fa fa-lock" ng-style="{color: showPermissions ? '#3071A9' : 'black' }"></i>
|
||||
</button>
|
||||
<i class="fa fa-lock" ng-style="{color: showPermissions ? '#3071A9' : 'black' }"></i>
|
||||
</span>
|
||||
|
||||
<span class="btn-group">
|
||||
|
|
|
|||
|
|
@ -223,6 +223,13 @@
|
|||
padding-left: 5px;
|
||||
}
|
||||
|
||||
.setting-btn {
|
||||
position: relative;
|
||||
top: 2px;
|
||||
margin-right: 4px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.cron-preset-container {
|
||||
padding: 10px 20px 0 20px;
|
||||
font-weight: normal;
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ angular.module('zeppelinWebApp')
|
|||
return notebook;
|
||||
}
|
||||
|
||||
if (notebook.children) {
|
||||
if (notebook.children) {
|
||||
filteringNote(notebook.children, filteredNotes);
|
||||
}
|
||||
});
|
||||
|
|
@ -63,7 +63,6 @@ angular.module('zeppelinWebApp')
|
|||
}, 500);
|
||||
};
|
||||
|
||||
|
||||
var vm = this;
|
||||
vm.notes = notebookListDataFactory;
|
||||
vm.connected = websocketMsgSrv.isConnected();
|
||||
|
|
@ -71,13 +70,6 @@ angular.module('zeppelinWebApp')
|
|||
vm.arrayOrderingSrv = arrayOrderingSrv;
|
||||
$scope.searchForm = searchService;
|
||||
|
||||
if ($rootScope.ticket) {
|
||||
$rootScope.fullUsername = $rootScope.ticket.principal;
|
||||
$rootScope.truncatedUsername = $rootScope.ticket.principal;
|
||||
}
|
||||
|
||||
var MAX_USERNAME_LENGTH=16;
|
||||
|
||||
angular.element('#notebook-list').perfectScrollbar({suppressScrollX: true});
|
||||
|
||||
$scope.$on('setNoteMenu', function(event, notes) {
|
||||
|
|
@ -88,43 +80,36 @@ angular.module('zeppelinWebApp')
|
|||
vm.connected = param;
|
||||
});
|
||||
|
||||
$scope.checkUsername = function () {
|
||||
if ($rootScope.ticket) {
|
||||
if ($rootScope.ticket.principal.length <= MAX_USERNAME_LENGTH) {
|
||||
$rootScope.truncatedUsername = $rootScope.ticket.principal;
|
||||
}
|
||||
else {
|
||||
$rootScope.truncatedUsername = $rootScope.ticket.principal.substr(0, MAX_USERNAME_LENGTH) + '..';
|
||||
}
|
||||
}
|
||||
if (_.isEmpty($rootScope.truncatedUsername)) {
|
||||
$rootScope.truncatedUsername = 'Connected';
|
||||
}
|
||||
};
|
||||
|
||||
$scope.$on('loginSuccess', function(event, param) {
|
||||
$scope.checkUsername();
|
||||
loadNotes();
|
||||
});
|
||||
|
||||
$scope.logout = function() {
|
||||
$http.post(baseUrlSrv.getRestApiBase()+'/login/logout')
|
||||
.success(function(data, status, headers, config) {
|
||||
$rootScope.userName = '';
|
||||
$rootScope.ticket.principal = '';
|
||||
$rootScope.ticket.ticket = '';
|
||||
$rootScope.ticket.roles = '';
|
||||
BootstrapDialog.show({
|
||||
message: 'Logout Success'
|
||||
});
|
||||
setTimeout(function() {
|
||||
window.location = '#';
|
||||
window.location.reload();
|
||||
}, 1000);
|
||||
}).
|
||||
error(function(data, status, headers, config) {
|
||||
console.log('Error %o %o', status, data.message);
|
||||
});
|
||||
var logoutURL = baseUrlSrv.getRestApiBase() + '/login/logout';
|
||||
var request = new XMLHttpRequest();
|
||||
|
||||
//force authcBasic (if configured) to logout by setting credentials as false:false
|
||||
request.open('post', logoutURL, true, 'false', 'false');
|
||||
request.onreadystatechange = function() {
|
||||
if (request.readyState === 4) {
|
||||
if (request.status === 401 || request.status === 405) {
|
||||
$rootScope.userName = '';
|
||||
$rootScope.ticket.principal = '';
|
||||
$rootScope.ticket.ticket = '';
|
||||
$rootScope.ticket.roles = '';
|
||||
BootstrapDialog.show({
|
||||
message: 'Logout Success'
|
||||
});
|
||||
setTimeout(function() {
|
||||
window.location.replace('/');
|
||||
}, 1000);
|
||||
} else {
|
||||
request.open('post', logoutURL, true, 'false', 'false');
|
||||
request.send();
|
||||
}
|
||||
}
|
||||
};
|
||||
request.send();
|
||||
};
|
||||
|
||||
$scope.search = function(searchTerm) {
|
||||
|
|
@ -146,7 +131,6 @@ angular.module('zeppelinWebApp')
|
|||
};
|
||||
|
||||
function getZeppelinVersion() {
|
||||
console.log('version');
|
||||
$http.get(baseUrlSrv.getRestApiBase() + '/version').success(
|
||||
function(data, status, headers, config) {
|
||||
$rootScope.zeppelinVersion = data.body;
|
||||
|
|
@ -161,6 +145,5 @@ angular.module('zeppelinWebApp')
|
|||
|
||||
getZeppelinVersion();
|
||||
vm.loadNotes();
|
||||
$scope.checkUsername();
|
||||
|
||||
});
|
||||
|
|
|
|||
|
|
@ -44,24 +44,15 @@ limitations under the License.
|
|||
<li class="divider"></li>
|
||||
<div id="notebook-list" class="scrollbar-container">
|
||||
<li class="filter-names" ng-include="'components/filterNoteNames/filter-note-names.html'"></li>
|
||||
<li ng-repeat="note in navbar.notes.root.children |notebookFilter:query track by $index" ng-class="{'active' : navbar.isActive(note.id)}" ng-include="'notebook_list_renderer.html'"></li>
|
||||
<li ng-repeat="note in navbar.notes.root.children |notebookFilter:query track by $index"
|
||||
ng-class="{'active' : navbar.isActive(note.id)}" ng-include="'notebook_list_renderer.html'"></li>
|
||||
</div>
|
||||
</ul>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#/interpreter">Interpreter</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#/credential">Credential</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#/configuration">Configuration</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
<ul class="nav navbar-nav navbar-right" style="margin-right:5px;">
|
||||
<li ng-if="ticket" style="margin-top:10px;">
|
||||
<li class="nav-component" ng-if="ticket">
|
||||
<!--TODO(bzz): move to Typeahead https://angular-ui.github.io/bootstrap -->
|
||||
|
||||
<form role="search" data-ng-model="searchForm"
|
||||
|
|
@ -89,43 +80,35 @@ limitations under the License.
|
|||
</span>
|
||||
</div>
|
||||
</form>
|
||||
<li class="nav-component">
|
||||
<i ng-if="navbar.connected" class="fa fa-circle server-connected"
|
||||
tooltip="WebSocket Connected" tooltip-placement="bottom"></i>
|
||||
<i ng-if="!navbar.connected" class="fa fa-circle server-disconnected"
|
||||
tooltip="WebSocket Disconnected" tooltip-placement="bottom"></i>
|
||||
</li>
|
||||
<li class="dropdown " dropdown="">
|
||||
<a class="dropdown-toggle" type="button" data-toggle="dropdown" href="#"
|
||||
ng-show="navbar.connected" ng-if="ticket.principal == 'anonymous' ">
|
||||
<i class="fa fa-circle server-status"
|
||||
ng-class="{'server-connected':navbar.connected, 'server-disconnected':!navbar.connected}"></i>
|
||||
Connected
|
||||
<span class="caret"></span>
|
||||
</a>
|
||||
<a class="dropdown-toggle" type="button" data-toggle="dropdown" href="#"
|
||||
ng-show="navbar.connected" ng-if="ticket.principal != 'anonymous' "
|
||||
tooltip-placement="bottom" tooltip="{{fullUsername}}">
|
||||
<i class="fa fa-circle server-status"
|
||||
ng-class="{'server-connected':navbar.connected, 'server-disconnected':!navbar.connected}"></i>
|
||||
{{truncatedUsername}}
|
||||
<span class="caret"></span></a>
|
||||
<a class="dropdown-toggle" type="button" data-toggle="dropdown" href="#"
|
||||
ng-show="!navbar.connected">
|
||||
<i class="fa fa-circle server-status"
|
||||
ng-class="{'server-connected':navbar.connected, 'server-disconnected':!navbar.connected}"></i>
|
||||
Disconnected
|
||||
<span class="caret"></span></a>
|
||||
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="" data-toggle="modal" data-target="#aboutModal">
|
||||
<i style="font-size: 15px;" class="fa fa-info"></i> About</a></li>
|
||||
<li ng-show="ticket.principal && ticket.principal!='anonymous'" style="left: 5px;">
|
||||
<a ng-click="logout()">Logout</a>
|
||||
</li>
|
||||
</ul>
|
||||
<li>
|
||||
<div class="dropdown">
|
||||
<button ng-if="ticket" class="nav-btn dropdown-toggle" type="button" data-toggle="dropdown" style="margin:11px 5px 0 0;">
|
||||
<span class="username">{{ticket.principal}}</span>
|
||||
<span class="caret" style="margin-bottom: 8px"></span>
|
||||
</button>
|
||||
<span ng-if="!ticket" style="margin: 5px;"></span>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="" data-toggle="modal" data-target="#aboutModal">About Zeppelin</a>
|
||||
<li role="separator" style="margin: 5px 0;" class="divider"></li>
|
||||
<li><a href="#/interpreter">Interpreter</a></li>
|
||||
<li><a href="#/credential">Credential</a></li>
|
||||
<li><a href="#/configuration">Configuration</a></li>
|
||||
<li ng-if="ticket.principal && ticket.principal !== 'anonymous'" role="separator" style="margin: 5px 0;" class="divider"></li>
|
||||
<li ng-if="ticket.principal && ticket.principal !== 'anonymous'"><a ng-click="logout()">Logout</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</li>
|
||||
<li ng-if="!ticket">
|
||||
<button class="btn btn-default" data-toggle="modal" data-target="#loginModal"
|
||||
ng-click="showLoginWindow()" style="margin-top: 8px">Login
|
||||
<li class="nav-component" ng-if="!ticket">
|
||||
<button class="btn nav-login-btn" data-toggle="modal" data-target="#loginModal"
|
||||
ng-click="showLoginWindow()">Login
|
||||
</button>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -346,6 +346,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
|
|||
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
|
||||
}
|
||||
|
||||
public String getInterpreterListPath() {
|
||||
return getRelativeDir(String.format("%s/interpreter-list", getConfDir()));
|
||||
}
|
||||
|
||||
public String getInterpreterDir() {
|
||||
return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -361,15 +361,17 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
List<Dependency> deps = intSetting.getDependencies();
|
||||
if (deps != null) {
|
||||
for (Dependency d: deps) {
|
||||
File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
|
||||
|
||||
if (d.getExclusions() != null) {
|
||||
depResolver.load(
|
||||
d.getGroupArtifactVersion(),
|
||||
d.getExclusions(),
|
||||
conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id());
|
||||
new File(destDir, intSetting.id()));
|
||||
} else {
|
||||
depResolver.load(
|
||||
d.getGroupArtifactVersion(),
|
||||
conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id());
|
||||
new File(destDir, intSetting.id()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,301 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.zeppelin.interpreter.install;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.log4j.ConsoleAppender;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.apache.zeppelin.conf.ZeppelinConfiguration;
|
||||
import org.apache.zeppelin.dep.DependencyResolver;
|
||||
import org.apache.zeppelin.util.Util;
|
||||
import org.sonatype.aether.RepositoryException;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Commandline utility to install interpreter from maven repository
|
||||
*/
|
||||
public class InstallInterpreter {
|
||||
private final File interpreterListFile;
|
||||
private final File interpreterBaseDir;
|
||||
private final List<AvailableInterpreterInfo> availableInterpreters;
|
||||
private final String localRepoDir;
|
||||
private URL proxyUrl;
|
||||
private String proxyUser;
|
||||
private String proxyPassword;
|
||||
|
||||
/**
|
||||
*
|
||||
* @param interpreterListFile
|
||||
* @param interpreterBaseDir interpreter directory for installing binaries
|
||||
* @throws IOException
|
||||
*/
|
||||
public InstallInterpreter(File interpreterListFile, File interpreterBaseDir, String localRepoDir)
|
||||
throws IOException {
|
||||
this.interpreterListFile = interpreterListFile;
|
||||
this.interpreterBaseDir = interpreterBaseDir;
|
||||
this.localRepoDir = localRepoDir;
|
||||
availableInterpreters = new LinkedList<AvailableInterpreterInfo>();
|
||||
readAvailableInterpreters();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Information for available informations
|
||||
*/
|
||||
private static class AvailableInterpreterInfo {
|
||||
public final String name;
|
||||
public final String artifact;
|
||||
public final String description;
|
||||
|
||||
public AvailableInterpreterInfo(String name, String artifact, String description) {
|
||||
this.name = name;
|
||||
this.artifact = artifact;
|
||||
this.description = description;
|
||||
}
|
||||
}
|
||||
|
||||
private void readAvailableInterpreters() throws IOException {
|
||||
if (!interpreterListFile.isFile()) {
|
||||
System.err.println("Can't find interpreter list " + interpreterListFile.getAbsolutePath());
|
||||
return;
|
||||
}
|
||||
String text = FileUtils.readFileToString(interpreterListFile);
|
||||
String[] lines = text.split("\n");
|
||||
|
||||
Pattern pattern = Pattern.compile("(\\S+)\\s+(\\S+)\\s+(.*)");
|
||||
|
||||
int lineNo = 0;
|
||||
for (String line : lines) {
|
||||
lineNo++;
|
||||
if (line == null || line.length() == 0 || line.startsWith("#")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Matcher match = pattern.matcher(line);
|
||||
if (match.groupCount() != 3) {
|
||||
System.err.println("Error on line " + lineNo + ", " + line);
|
||||
continue;
|
||||
}
|
||||
|
||||
match.find();
|
||||
|
||||
String name = match.group(1);
|
||||
String artifact = match.group(2);
|
||||
String description = match.group(3);
|
||||
|
||||
availableInterpreters.add(new AvailableInterpreterInfo(name, artifact, description));
|
||||
}
|
||||
}
|
||||
|
||||
public List<AvailableInterpreterInfo> list() {
|
||||
for (AvailableInterpreterInfo info : availableInterpreters) {
|
||||
System.out.println(info.name + "\t\t\t" + info.description);
|
||||
}
|
||||
|
||||
return availableInterpreters;
|
||||
}
|
||||
|
||||
public void installAll() {
|
||||
for (AvailableInterpreterInfo info : availableInterpreters) {
|
||||
install(info.name, info.artifact);
|
||||
}
|
||||
}
|
||||
|
||||
public void install(String [] names) {
|
||||
for (String name : names) {
|
||||
install(name);
|
||||
}
|
||||
}
|
||||
|
||||
public void install(String name) {
|
||||
// find artifact name
|
||||
for (AvailableInterpreterInfo info : availableInterpreters) {
|
||||
if (name.equals(info.name)) {
|
||||
install(name, info.artifact);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
throw new RuntimeException("Can't find interpreter '" + name + "'");
|
||||
}
|
||||
|
||||
public void install(String [] names, String [] artifacts) {
|
||||
if (names.length != artifacts.length) {
|
||||
throw new RuntimeException("Length of given names and artifacts are different");
|
||||
}
|
||||
|
||||
for (int i = 0; i < names.length; i++) {
|
||||
install(names[i], artifacts[i]);
|
||||
}
|
||||
}
|
||||
|
||||
public void install(String name, String artifact) {
|
||||
DependencyResolver depResolver = new DependencyResolver(localRepoDir);
|
||||
if (proxyUrl != null) {
|
||||
depResolver.setProxy(proxyUrl, proxyUser, proxyPassword);
|
||||
}
|
||||
|
||||
File installDir = new File(interpreterBaseDir, name);
|
||||
if (installDir.exists()) {
|
||||
System.err.println("Directory " + installDir.getAbsolutePath() + " already exists. Skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
System.out.println("Install " + name + "(" + artifact + ") to "
|
||||
+ installDir.getAbsolutePath() + " ... ");
|
||||
|
||||
try {
|
||||
depResolver.load(artifact, installDir);
|
||||
System.out.println("Interpreter " + name + " installed under " +
|
||||
installDir.getAbsolutePath() + ".");
|
||||
} catch (RepositoryException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) {
|
||||
this.proxyUrl = proxyUrl;
|
||||
this.proxyUser = proxyUser;
|
||||
this.proxyPassword = proxyPassword;
|
||||
}
|
||||
|
||||
public static void usage() {
|
||||
System.out.println("Options");
|
||||
System.out.println(" -l, --list List available interpreters");
|
||||
System.out.println(" -a, --all Install all available interpreters");
|
||||
System.out.println(" -n, --name [NAMES] Install interpreters (comma separated " +
|
||||
"list)" +
|
||||
"e.g. md,shell,jdbc,python,angular");
|
||||
System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names" +
|
||||
". " +
|
||||
"(comma separated list correspond to --name) " +
|
||||
"e.g. customGroup:customArtifact:customVersion");
|
||||
System.out.println(" --proxy-url [url] (Optional) proxy url. http(s)://host:port");
|
||||
System.out.println(" --proxy-user [user] (Optional) proxy user");
|
||||
System.out.println(" --proxy-password [password] (Optional) proxy password");
|
||||
}
|
||||
|
||||
public static void main(String [] args) throws IOException {
|
||||
if (args.length == 0) {
|
||||
usage();
|
||||
return;
|
||||
}
|
||||
|
||||
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
|
||||
InstallInterpreter installer = new InstallInterpreter(
|
||||
new File(conf.getInterpreterListPath()),
|
||||
new File(conf.getInterpreterDir()),
|
||||
conf.getInterpreterLocalRepoPath());
|
||||
|
||||
String names = null;
|
||||
String artifacts = null;
|
||||
URL proxyUrl = null;
|
||||
String proxyUser = null;
|
||||
String proxyPassword = null;
|
||||
boolean all = false;
|
||||
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
String arg = args[i].toLowerCase(Locale.US);
|
||||
switch (arg) {
|
||||
case "--list":
|
||||
case "-l":
|
||||
installer.list();
|
||||
System.exit(0);
|
||||
break;
|
||||
case "--all":
|
||||
case "-a":
|
||||
all = true;
|
||||
break;
|
||||
case "--name":
|
||||
case "-n":
|
||||
names = args[++i];
|
||||
break;
|
||||
case "--artifact":
|
||||
case "-t":
|
||||
artifacts = args[++i];
|
||||
break;
|
||||
case "--version":
|
||||
case "-v":
|
||||
Util.getVersion();
|
||||
break;
|
||||
case "--proxy-url":
|
||||
proxyUrl = new URL(args[++i]);
|
||||
break;
|
||||
case "--proxy-user":
|
||||
proxyUser = args[++i];
|
||||
break;
|
||||
case "--proxy-password":
|
||||
proxyPassword = args[++i];
|
||||
break;
|
||||
case "--help":
|
||||
case "-h":
|
||||
usage();
|
||||
System.exit(0);
|
||||
break;
|
||||
default:
|
||||
System.out.println("Unknown option " + arg);
|
||||
}
|
||||
}
|
||||
|
||||
if (proxyUrl != null) {
|
||||
installer.setProxy(proxyUrl, proxyUser, proxyPassword);
|
||||
}
|
||||
|
||||
if (all) {
|
||||
installer.installAll();
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
if (names != null) {
|
||||
if (artifacts != null) {
|
||||
installer.install(names.split(","), artifacts.split(","));
|
||||
startTip();
|
||||
configurationTip();
|
||||
interpreterSettingTip();
|
||||
} else {
|
||||
installer.install(names.split(","));
|
||||
startTip();
|
||||
interpreterSettingTip();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void startTip() {
|
||||
System.out.println("");
|
||||
}
|
||||
|
||||
private static void configurationTip() {
|
||||
System.out.println("Add interpreter class name to '"
|
||||
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName() + "' property "
|
||||
+ "in your conf/zeppelin-site.xml file");
|
||||
}
|
||||
|
||||
private static void interpreterSettingTip() {
|
||||
System.out.println("Create interpreter setting in 'Interpreter' menu on GUI."
|
||||
+ " And then you can bind interpreter on your notebook");
|
||||
}
|
||||
}
|
||||
|
|
@ -205,7 +205,9 @@ public class Notebook implements NoteEventListener {
|
|||
throws IOException {
|
||||
GsonBuilder gsonBuilder = new GsonBuilder();
|
||||
gsonBuilder.setPrettyPrinting();
|
||||
Gson gson = gsonBuilder.create();
|
||||
|
||||
Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer())
|
||||
.create();
|
||||
JsonReader reader = new JsonReader(new StringReader(sourceJson));
|
||||
reader.setLenient(true);
|
||||
Note newNote;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.notebook;
|
||||
|
||||
import com.google.gson.JsonDeserializationContext;
|
||||
import com.google.gson.JsonDeserializer;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonParseException;
|
||||
|
||||
import java.lang.reflect.Type;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* importNote date format deserializer
|
||||
*/
|
||||
public class NotebookImportDeserializer implements JsonDeserializer<Date> {
|
||||
private static final String[] DATE_FORMATS = new String[] {
|
||||
"yyyy-MM-dd'T'HH:mm:ssZ",
|
||||
"MMM dd, yyyy HH:mm:ss"
|
||||
};
|
||||
|
||||
@Override
|
||||
public Date deserialize(JsonElement jsonElement, Type typeOF,
|
||||
JsonDeserializationContext context) throws JsonParseException {
|
||||
for (String format : DATE_FORMATS) {
|
||||
try {
|
||||
return new SimpleDateFormat(format, Locale.US).parse(jsonElement.getAsString());
|
||||
} catch (ParseException e) {
|
||||
}
|
||||
}
|
||||
throw new JsonParseException("Unparsable date: \"" + jsonElement.getAsString()
|
||||
+ "\". Supported formats: " + Arrays.toString(DATE_FORMATS));
|
||||
}
|
||||
}
|
||||
|
|
@ -18,13 +18,8 @@
|
|||
package org.apache.zeppelin.util;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
|
|
@ -44,162 +39,6 @@ public class Util {
|
|||
}
|
||||
}
|
||||
|
||||
public static String[] split(String str, char split) {
|
||||
return split(str, new String[] {String.valueOf(split)}, false);
|
||||
}
|
||||
|
||||
public static String[] split(String str, String[] splitters, boolean includeSplitter) {
|
||||
String escapeSeq = "\"',;<%>";
|
||||
char escapeChar = '\\';
|
||||
String[] blockStart = new String[] {"\"", "'", "<%", "N_<"};
|
||||
String[] blockEnd = new String[] {"\"", "'", "%>", "N_>"};
|
||||
|
||||
return split(str, escapeSeq, escapeChar, blockStart, blockEnd, splitters, includeSplitter);
|
||||
|
||||
}
|
||||
|
||||
public static String[] split(String str, String escapeSeq, char escapeChar, String[] blockStart,
|
||||
String[] blockEnd, String[] splitters, boolean includeSplitter) {
|
||||
|
||||
List<String> splits = new ArrayList<String>();
|
||||
|
||||
String curString = "";
|
||||
|
||||
boolean escape = false; // true when escape char is found
|
||||
int lastEscapeOffset = -1;
|
||||
int blockStartPos = -1;
|
||||
List<Integer> blockStack = new LinkedList<Integer>();
|
||||
|
||||
for (int i = 0; i < str.length(); i++) {
|
||||
char c = str.charAt(i);
|
||||
|
||||
// escape char detected
|
||||
if (c == escapeChar && escape == false) {
|
||||
escape = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
// escaped char comes
|
||||
if (escape == true) {
|
||||
if (escapeSeq.indexOf(c) < 0) {
|
||||
curString += escapeChar;
|
||||
}
|
||||
curString += c;
|
||||
escape = false;
|
||||
lastEscapeOffset = curString.length();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (blockStack.size() > 0) { // inside of block
|
||||
curString += c;
|
||||
// check multichar block
|
||||
boolean multicharBlockDetected = false;
|
||||
for (int b = 0; b < blockStart.length; b++) {
|
||||
if (blockStartPos >= 0
|
||||
&& getBlockStr(blockStart[b]).compareTo(str.substring(blockStartPos, i)) == 0) {
|
||||
blockStack.remove(0);
|
||||
blockStack.add(0, b);
|
||||
multicharBlockDetected = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (multicharBlockDetected == true) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// check if current block is nestable
|
||||
if (isNestedBlock(blockStart[blockStack.get(0)]) == true) {
|
||||
// try to find nested block start
|
||||
|
||||
if (curString.substring(lastEscapeOffset + 1).endsWith(
|
||||
getBlockStr(blockStart[blockStack.get(0)])) == true) {
|
||||
blockStack.add(0, blockStack.get(0)); // block is started
|
||||
blockStartPos = i;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// check if block is finishing
|
||||
if (curString.substring(lastEscapeOffset + 1).endsWith(
|
||||
getBlockStr(blockEnd[blockStack.get(0)]))) {
|
||||
// the block closer is one of the splitters (and not nested block)
|
||||
if (isNestedBlock(blockEnd[blockStack.get(0)]) == false) {
|
||||
for (String splitter : splitters) {
|
||||
if (splitter.compareTo(getBlockStr(blockEnd[blockStack.get(0)])) == 0) {
|
||||
splits.add(curString);
|
||||
if (includeSplitter == true) {
|
||||
splits.add(splitter);
|
||||
}
|
||||
curString = "";
|
||||
lastEscapeOffset = -1;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
blockStartPos = -1;
|
||||
blockStack.remove(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
} else { // not in the block
|
||||
boolean splitted = false;
|
||||
for (String splitter : splitters) {
|
||||
// forward check for splitter
|
||||
if (splitter.compareTo(
|
||||
str.substring(i, Math.min(i + splitter.length(), str.length()))) == 0) {
|
||||
splits.add(curString);
|
||||
if (includeSplitter == true) {
|
||||
splits.add(splitter);
|
||||
}
|
||||
curString = "";
|
||||
lastEscapeOffset = -1;
|
||||
i += splitter.length() - 1;
|
||||
splitted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (splitted == true) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// add char to current string
|
||||
curString += c;
|
||||
|
||||
// check if block is started
|
||||
for (int b = 0; b < blockStart.length; b++) {
|
||||
if (curString.substring(lastEscapeOffset + 1)
|
||||
.endsWith(getBlockStr(blockStart[b])) == true) {
|
||||
blockStack.add(0, b); // block is started
|
||||
blockStartPos = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (curString.length() > 0) {
|
||||
splits.add(curString.trim());
|
||||
}
|
||||
return splits.toArray(new String[] {});
|
||||
|
||||
}
|
||||
|
||||
private static String getBlockStr(String blockDef) {
|
||||
if (blockDef.startsWith("N_")) {
|
||||
return blockDef.substring("N_".length());
|
||||
} else {
|
||||
return blockDef;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean isNestedBlock(String blockDef) {
|
||||
if (blockDef.startsWith("N_")) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Zeppelin version
|
||||
*
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
package org.apache.zeppelin.interpreter.install;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.zeppelin.conf.ZeppelinConfiguration;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
public class InstallInterpreterTest {
|
||||
private File tmpDir;
|
||||
private InstallInterpreter installer;
|
||||
private File interpreterBaseDir;
|
||||
|
||||
@Before
|
||||
public void setUp() throws IOException {
|
||||
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
|
||||
new File(tmpDir, "conf").mkdirs();
|
||||
interpreterBaseDir = new File(tmpDir, "interpreter");
|
||||
File localRepoDir = new File(tmpDir, "local-repo");
|
||||
interpreterBaseDir.mkdir();
|
||||
localRepoDir.mkdir();
|
||||
|
||||
File interpreterListFile = new File(tmpDir, "conf/interpreter-list");
|
||||
|
||||
|
||||
// create interpreter list file
|
||||
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
|
||||
|
||||
String interpreterList = "";
|
||||
interpreterList += "intp1 org.apache.commons:commons-csv:1.1 test interpreter 1\n";
|
||||
interpreterList += "intp2 org.apache.commons:commons-math3:3.6.1 test interpreter 2\n";
|
||||
|
||||
FileUtils.writeStringToFile(new File(tmpDir, "conf/interpreter-list"), interpreterList);
|
||||
|
||||
installer = new InstallInterpreter(interpreterListFile, interpreterBaseDir, localRepoDir
|
||||
.getAbsolutePath());
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws IOException {
|
||||
FileUtils.deleteDirectory(tmpDir);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testList() {
|
||||
assertEquals(2, installer.list().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void install() {
|
||||
assertEquals(0, interpreterBaseDir.listFiles().length);
|
||||
|
||||
installer.install("intp1");
|
||||
assertTrue(new File(interpreterBaseDir, "intp1").isDirectory());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void installAll() {
|
||||
installer.installAll();
|
||||
assertTrue(new File(interpreterBaseDir, "intp1").isDirectory());
|
||||
assertTrue(new File(interpreterBaseDir, "intp2").isDirectory());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.util;
|
||||
|
||||
import org.apache.zeppelin.util.Util;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
public class UtilTest extends TestCase {
|
||||
|
||||
@Override
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void tearDown() throws Exception {
|
||||
super.tearDown();
|
||||
}
|
||||
|
||||
public void testSplitIncludingToken() {
|
||||
String[] token = Util.split("hello | \"world '>|hehe\" > next >> sink", new String[]{"|", ">>", ">"}, true);
|
||||
assertEquals(7, token.length);
|
||||
assertEquals(" \"world '>|hehe\" ", token[2]);
|
||||
}
|
||||
|
||||
public void testSplitExcludingToken() {
|
||||
String[] token = Util.split("hello | \"world '>|hehe\" > next >> sink", new String[]{"|", ">>", ">"}, false);
|
||||
assertEquals(4, token.length);
|
||||
assertEquals(" \"world '>|hehe\" ", token[1]);
|
||||
}
|
||||
|
||||
public void testSplitWithSemicolonEnd(){
|
||||
String[] token = Util.split("show tables;", ';');
|
||||
assertEquals(1, token.length);
|
||||
assertEquals("show tables", token[0]);
|
||||
}
|
||||
|
||||
public void testEscapeTemplate(){
|
||||
String[] token = Util.split("select * from <%=table%> limit 1 > output", '>');
|
||||
assertEquals(2, token.length);
|
||||
assertEquals("output", token[1]);
|
||||
}
|
||||
|
||||
public void testSplit(){
|
||||
String [] op = new String[]{";", "|", ">>", ">"};
|
||||
|
||||
String str = "CREATE external table news20b_train (\n"+
|
||||
" rowid int,\n"+
|
||||
" label int,\n"+
|
||||
" features ARRAY<STRING>\n"+
|
||||
")\n"+
|
||||
"ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' \n"+
|
||||
"COLLECTION ITEMS TERMINATED BY \",\" \n"+
|
||||
"STORED AS TEXTFILE;\n";
|
||||
Util.split(str, op, true);
|
||||
|
||||
}
|
||||
|
||||
public void testSplitDifferentBlockStartEnd(){
|
||||
String [] op = new String[]{";", "|", ">>", ">"};
|
||||
String escapeSeq = "\"',;<%>!";
|
||||
char escapeChar = '\\';
|
||||
String [] blockStart = new String[]{ "\"", "'", "<%", "<", "!"};
|
||||
String [] blockEnd = new String[]{ "\"", "'", "%>", ">", ";" };
|
||||
String [] t = Util.split("!echo a;!echo b;", escapeSeq, escapeChar, blockStart, blockEnd, op, true);
|
||||
assertEquals(4, t.length);
|
||||
assertEquals("!echo a;", t[0]);
|
||||
assertEquals(";", t[1]);
|
||||
assertEquals("!echo b;", t[2]);
|
||||
assertEquals(";", t[3]);
|
||||
}
|
||||
|
||||
public void testNestedBlock(){
|
||||
String [] op = new String[]{";", "|", ">>", ">"};
|
||||
String escapeSeq = "\"',;<%>!";
|
||||
char escapeChar = '\\';
|
||||
String [] blockStart = new String[]{ "\"", "'", "<%", "N_<", "<", "!"};
|
||||
String [] blockEnd = new String[]{ "\"", "'", "%>", "N_>", ";", ";" };
|
||||
String [] t = Util.split("array <STRUCT<STRING>> tags|aa", escapeSeq, escapeChar, blockStart, blockEnd, op, true);
|
||||
assertEquals(3, t.length);
|
||||
assertEquals("array <STRUCT<STRING>> tags", t[0]);
|
||||
assertEquals("aa", t[2]);
|
||||
}
|
||||
|
||||
public void testGetVersion(){
|
||||
String version = Util.getVersion();
|
||||
assertNotNull(version);
|
||||
System.out.println(version);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.util;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class UtilsForTests {
|
||||
|
||||
static Logger LOGGER = LoggerFactory.getLogger(UtilsForTests.class);
|
||||
|
||||
public static File createTmpDir() throws Exception {
|
||||
File tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
|
||||
tmpDir.mkdir();
|
||||
return tmpDir;
|
||||
|
||||
}
|
||||
/*
|
||||
private static final String HADOOP_DIST="http://apache.mirror.cdnetworks.com/hadoop/common/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz";
|
||||
//private static final String HADOOP_DIST="http://www.us.apache.org/dist/hadoop/common/hadoop-1.2.1/hadoop-1.2.1-bin.tar.gz";
|
||||
|
||||
public static void getHadoop() throws MalformedURLException, IOException{
|
||||
setEnv("HADOOP_HOME", new File("./target/hadoop-1.2.1").getAbsolutePath());
|
||||
if(new File("./target/hadoop-1.2.1").isDirectory()) return;
|
||||
//System.out.println("Downloading a hadoop distribution ... it will take a while");
|
||||
//FileUtils.copyURLToFile(new URL(HADOOP_DIST), new File("/tmp/zp_test_hadoop-bin.tar.gz"));
|
||||
System.out.println("Unarchive hadoop distribution ... ");
|
||||
new File("./target").mkdir();
|
||||
Runtime.getRuntime().exec("tar -xzf /tmp/zp_test_hadoop-bin.tar.gz -C ./target");
|
||||
}
|
||||
*/
|
||||
|
||||
public static void delete(File file) {
|
||||
if (file.isFile()) file.delete();
|
||||
else if (file.isDirectory()) {
|
||||
File[] files = file.listFiles();
|
||||
if (files != null && files.length > 0) {
|
||||
for (File f : files) {
|
||||
delete(f);
|
||||
}
|
||||
}
|
||||
file.delete();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method to create a file (if does not exist) and populate it the the given content
|
||||
*
|
||||
* @param path to file
|
||||
* @param content of the file
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void createFileWithContent(String path, String content) throws IOException {
|
||||
File f = new File(path);
|
||||
if (!f.exists()) {
|
||||
stringToFile(content, f);
|
||||
}
|
||||
}
|
||||
|
||||
public static void stringToFile(String string, File file) throws IOException {
|
||||
FileOutputStream out = new FileOutputStream(file);
|
||||
out.write(string.getBytes());
|
||||
out.close();
|
||||
}
|
||||
|
||||
@SuppressWarnings({"unchecked", "rawtypes"})
|
||||
public static void setEnv(String k, String v) {
|
||||
Map<String, String> newenv = new HashMap<String, String>();
|
||||
newenv.put(k, v);
|
||||
try {
|
||||
Class<?> processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment");
|
||||
Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment");
|
||||
theEnvironmentField.setAccessible(true);
|
||||
Map<String, String> env = (Map<String, String>) theEnvironmentField.get(null);
|
||||
env.putAll(newenv);
|
||||
Field theCaseInsensitiveEnvironmentField = processEnvironmentClass.getDeclaredField("theCaseInsensitiveEnvironment");
|
||||
theCaseInsensitiveEnvironmentField.setAccessible(true);
|
||||
Map<String, String> cienv = (Map<String, String>) theCaseInsensitiveEnvironmentField.get(null);
|
||||
cienv.putAll(newenv);
|
||||
} catch (NoSuchFieldException e) {
|
||||
try {
|
||||
Class[] classes = Collections.class.getDeclaredClasses();
|
||||
Map<String, String> env = System.getenv();
|
||||
for (Class cl : classes) {
|
||||
if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
|
||||
Field field = cl.getDeclaredField("m");
|
||||
field.setAccessible(true);
|
||||
Object obj = field.get(env);
|
||||
Map<String, String> map = (Map<String, String>) obj;
|
||||
map.clear();
|
||||
map.putAll(newenv);
|
||||
}
|
||||
}
|
||||
} catch (Exception e2) {
|
||||
LOGGER.error(e2.toString(), e2);
|
||||
}
|
||||
} catch (Exception e1) {
|
||||
LOGGER.error(e1.toString(), e1);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue