Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into ZEPPELIN-212

Browse source
This commit is contained in:
Lee moon soo 2016-11-24 10:31:16 -08:00
commit 0f1d28f022
117 changed files with 5945 additions and 735 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.travis.yml
View file

@ -18,10 +18,14 @@ language: java
sudo: false
cache:
apt: true
directories:
- .spark-dist
- ${HOME}/.m2/repository/.cache/maven-download-plugin
- .node_modules
- ${HOME}/.m2
- ${HOME}/R
- zeppelin-web/node
- zeppelin-web/node_modules
- zeppelin-web/bower_components
addons:
apt:
@ -63,6 +67,7 @@ matrix:
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
before_install:
- echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit'" >> ~/.mavenrc
- "ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin"
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
- mkdir -p ~/R
@ -73,7 +78,7 @@ before_install:
- ./dev/change_scala_version.sh $SCALA_VER
install:
- mvn $BUILD_FLAG $PROFILE -B
- mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $BUILD_FLAG $PROFILE -B
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
@ -81,8 +86,7 @@ before_script:
- tail conf/zeppelin-env.sh
script:
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
- rm -rf .node_modules; cp -r zeppelin-web/node_modules .node_modules
- mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=warn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
after_success:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"

2484
_tools/maven-4.0.0.xsd Normal file
View file

File diff suppressed because it is too large Load diff

2
beam/src/main/test/org/apache/zeppelin/beam/BeamInterpreterTest.java
View file

@ -43,7 +43,7 @@ public class BeamInterpreterTest {
Properties p = new Properties();
beam = new BeamInterpreter(p);
beam.open();
context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null,
context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null,
null);
}

1
conf/zeppelin-env.sh.template
View file

@ -38,6 +38,7 @@
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth?
# export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise
#### Spark interpreter configuration ####

9
conf/zeppelin-site.xml.template
View file

@ -190,7 +190,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>
@ -277,6 +277,12 @@
<description>Anonymous user allowed by default</description>
</property>
<property>
<name>zeppelin.notebook.public</name>
<value>true</value>
<description>Make notebook public by default when created, private otherwise</description>
</property>
<property>
<name>zeppelin.websocket.max.text.message.size</name>
<value>1024000</value>
@ -284,4 +290,3 @@
</property>
</configuration>

BIN
docs/assets/themes/zeppelin/img/docs-img/add_credential.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 164 KiB

After

Width:  |  Height:  |  Size: 56 KiB

12
docs/install/install.md
View file

@ -87,7 +87,7 @@ Congratulations, you have successfully installed Apache Zeppelin! Here are few s
* For an in-depth overview, head to [Explore Apache Zeppelin UI](../quickstart/explorezeppelinui.html).
* And then, try run [tutorial](http://localhost:8080/#/notebook/2A94M5J1Z) notebook in your Zeppelin.
* And see how to change [configurations](#apache-zeppelin-configuration) like port number, etc.
#### Zeppelin with Apache Spark ...
* To know more about deep integration with [Apache Spark](http://spark.apache.org/), check [Spark Interpreter](../interpreter/spark.html).
@ -95,7 +95,7 @@ Congratulations, you have successfully installed Apache Zeppelin! Here are few s
* Check [JDBC Interpreter](../interpreter/jdbc.html) to know more about configure and uses multiple JDBC data sources.
#### Zeppelin with Python ...
* Check [Python interpreter](../interpreter/python.html) to know more about Matplotlib, Pandas integration.
* Check [Python interpreter](../interpreter/python.html) to know more about Matplotlib, Pandas, Conda/Docker environment integration.
#### Multi-user environment ...
@ -306,6 +306,12 @@ You can configure Apache Zeppelin with either **environment variables** in `conf
<td>false</td>
<td>If there are multiple notebook storage locations, should we treat the first one as the only source of truth?</td>
</tr>
<tr>
<td>ZEPPELIN_NOTEBOOK_PUBLIC</td>
<td>zeppelin.notebook.public</td>
<td>true</td>
<td>Make notebook public (set only `owners`) by default when created/imported. If set to `false` will add `user` to `readers` and `writers` as well, making it private and invisible to other users unless permissions are granted.</td>
</tr>
<tr>
<td>ZEPPELIN_INTERPRETERS</td>
<td>zeppelin.interpreters</td>
@ -377,4 +383,4 @@ exec bin/zeppelin-daemon.sh upstart
## Building from Source
If you want to build from source instead of using binary package, follow the instructions [here](./build.html).
If you want to build from source instead of using binary package, follow the instructions [here](./build.html).

6
docs/interpreter/jdbc.md
View file

@ -116,7 +116,11 @@ The JDBC interpreter properties are defined by default like below.
</tr>
</table>
If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.
If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.
You can also use [Credential](../security/datasource_authorization.html) for JDBC authentication.
If `default.user` and `default.password` properties are deleted(using X button) for database connection in the interpreter setting page,
the JDBC interpreter will get the account information from [Credential](../security/datasource_authorization.html).
The below example is for `Mysql` connection.
<img src="../assets/themes/zeppelin/img/docs-img/edit_properties.png" width="600px" />

56
docs/interpreter/python.md
View file

@ -56,9 +56,63 @@ The interpreter can only work if you already have python installed (the interpre
To access the help, type **help()**
## Python modules
## Python environments
### Default
By default, PythonInterpreter will use python command defined in `zeppelin.python` property to run python process.
The interpreter can use all modules already installed (with pip, easy_install...)
### Conda
[Conda](http://conda.pydata.org/) is an package management system and environment management system for python.
`%python.conda` interpreter lets you change between environments.
#### Usage
List your environments
```
%python.conda
```
Activate an environment
```
%python.conda activate [ENVIRONMENT_NAME]
```
Deactivate
```
%python.conda deactivate
```
### Docker
`%python.docker` interpreter allows PythonInterpreter creates python process in a specified docker container.
#### Usage
Activate an environment
```
%python.docker activate [Repository]
%python.docker activate [Repository:Tag]
%python.docker activate [Image Id]
```
Deactivate
```
%python.docker deactivate
```
Example
```
# activate latest tensorflow image as a python environment
%python.docker activate gcr.io/tensorflow/tensorflow:latest
```
## Using Zeppelin Dynamic Forms
You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your Python code.

3
docs/interpreter/spark.md
View file

@ -228,8 +228,7 @@ Here are few examples:
```
### 3. Dynamic Dependency Loading via %spark.dep interpreter
> Note: `%spark.dep` interpreter is deprecated since v0.6.0.
`%spark.dep` interpreter loads libraries to `%spark` and `%spark.pyspark` but not to `%spark.sql` interpreter. So we recommend you to use the first option instead.
> Note: `%spark.dep` interpreter loads libraries to `%spark` and `%spark.pyspark` but not to `%spark.sql` interpreter. So we recommend you to use the first option instead.
When your code requires external library, instead of doing download/copy/restart Zeppelin, you can easily do following jobs using `%spark.dep` interpreter.

59
docs/rest-api/rest-interpreter.md
View file

@ -178,6 +178,65 @@ The role of registered interpreters, settings and interpreters group are describ
</td>
</tr>
</table>
<br/>
### Get a registered interpreter setting by the setting id
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```GET``` method returns a registered interpreter setting on the server.</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/interpreter/setting/[setting ID]```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td>Fail code</td>
<td>
400 if such interpreter setting id does not exist <br/>
500 for any other errors
</td>
</tr>
<tr>
<td>Sample JSON response</td>
<td>
<pre>
{
"status": "OK",
"message": "",
"body": {
"id": "2AYW25ANY",
"name": "Markdown setting name",
"group": "md",
"properties": {
"propname": "propvalue"
},
"interpreterGroup": [
{
"class": "org.apache.zeppelin.markdown.Markdown",
"name": "md"
}
],
"dependencies": [
{
"groupArtifactVersion": "groupId:artifactId:version",
"exclusions": [
"groupId:artifactId"
]
}
]
}
}
</pre>
</td>
</tr>
</table>
<br/>
### Create a new interpreter setting

5
docs/security/datasource_authorization.md
View file

@ -37,7 +37,10 @@ You can add new credentials in the dropdown menu for your data source which can
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/credential_tab.png" width="180px"/>
**Entity** can be the key that distinguishes each credential sets. Type **Username & Password** for your own credentials. ex) user & password of Mysql
**Entity** can be the key that distinguishes each credential sets.(We suggest that the convention of the **Entity** is `[Interpreter Group].[Interpreter Name]`.)
Please see [what is interpreter group](../manual/interpreters.html#what-is-interpreter-group) for the detailed information.
Type **Username & Password** for your own credentials. ex) Mysql user & password of the JDBC Interpreter.
<img class="img-responsive" src="../assets/themes/zeppelin/img/docs-img/add_credential.png" />

2
flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class FlinkInterpreterTest {
Properties p = new Properties();
flink = new FlinkInterpreter(p);
flink.open();
context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
context = new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, null);
}
@AfterClass

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class IgniteInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, null);
private IgniteInterpreter intp;
private Ignite ignite;

2
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java
View file

@ -44,7 +44,7 @@ public class IgniteSqlInterpreterTest {
private static final String HOST = "127.0.0.1:47500..47509";
private static final InterpreterContext INTP_CONTEXT =
new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null);
new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, null);
private Ignite ignite;
private IgniteSqlInterpreter intp;

352
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -39,6 +39,8 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.jdbc.security.JDBCSecurityImpl;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.user.UserCredentials;
import org.apache.zeppelin.user.UsernamePassword;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -73,6 +75,9 @@ public class JDBCInterpreter extends Interpreter {
private Logger logger = LoggerFactory.getLogger(JDBCInterpreter.class);
static final String INTERPRETER_NAME = "jdbc";
static final String JDBC_DEFAULT_USER_KEY = "default.user";
static final String JDBC_DEFAULT_PASSWORD_KEY = "default.password";
static final String COMMON_KEY = "common";
static final String MAX_LINE_KEY = "max_count";
static final String MAX_LINE_DEFAULT = "1000";
@ -100,16 +105,12 @@ public class JDBCInterpreter extends Interpreter {
static final String EMPTY_COLUMN_VALUE = "";
private final String CONCURRENT_EXECUTION_KEY = "zeppelin.jdbc.concurrent.use";
private final String CONCURRENT_EXECUTION_COUNT = "zeppelin.jdbc.concurrent.max_connection";
private final String DBCP_STRING = "jdbc:apache:commons:dbcp:";
private final HashMap<String, Properties> propertiesMap;
private final Map<String, Statement> paragraphIdStatementMap;
private final Map<String, PoolingDriver> poolingDriverMap;
private final HashMap<String, Properties> basePropretiesMap;
private final HashMap<String, JDBCUserConfigurations> jdbcUserConfigurationsMap;
private final Map<String, SqlCompleter> propertyKeySqlCompleterMap;
private static final Function<CharSequence, InterpreterCompletion> sequenceToStringTransformer =
@ -123,14 +124,13 @@ public class JDBCInterpreter extends Interpreter {
public JDBCInterpreter(Properties property) {
super(property);
propertiesMap = new HashMap<>();
paragraphIdStatementMap = new HashMap<>();
poolingDriverMap = new HashMap<>();
jdbcUserConfigurationsMap = new HashMap<>();
propertyKeySqlCompleterMap = new HashMap<>();
basePropretiesMap = new HashMap<>();
}
public HashMap<String, Properties> getPropertiesMap() {
return propertiesMap;
return basePropretiesMap;
}
@Override
@ -140,21 +140,22 @@ public class JDBCInterpreter extends Interpreter {
String[] keyValue = propertyKey.split("\\.", 2);
if (2 == keyValue.length) {
logger.info("key: {}, value: {}", keyValue[0], keyValue[1]);
Properties prefixProperties;
if (propertiesMap.containsKey(keyValue[0])) {
prefixProperties = propertiesMap.get(keyValue[0]);
if (basePropretiesMap.containsKey(keyValue[0])) {
prefixProperties = basePropretiesMap.get(keyValue[0]);
} else {
prefixProperties = new Properties();
propertiesMap.put(keyValue[0], prefixProperties);
basePropretiesMap.put(keyValue[0], prefixProperties);
}
prefixProperties.put(keyValue[1], property.getProperty(propertyKey));
}
}
Set<String> removeKeySet = new HashSet<>();
for (String key : propertiesMap.keySet()) {
for (String key : basePropretiesMap.keySet()) {
if (!COMMON_KEY.equals(key)) {
Properties properties = propertiesMap.get(key);
Properties properties = basePropretiesMap.get(key);
if (!properties.containsKey(DRIVER_KEY) || !properties.containsKey(URL_KEY)) {
logger.error("{} will be ignored. {}.{} and {}.{} is mandatory.",
key, DRIVER_KEY, key, key, URL_KEY);
@ -164,15 +165,14 @@ public class JDBCInterpreter extends Interpreter {
}
for (String key : removeKeySet) {
propertiesMap.remove(key);
basePropretiesMap.remove(key);
}
logger.debug("propertiesMap: {}", propertiesMap);
logger.debug("JDBC PropretiesMap: {}", basePropretiesMap);
if (!StringUtils.isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
JDBCSecurityImpl.createSecureConfiguration(property);
}
for (String propertyKey : propertiesMap.keySet()) {
for (String propertyKey : basePropretiesMap.keySet()) {
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(null));
}
}
@ -194,128 +194,24 @@ public class JDBCInterpreter extends Interpreter {
return completer;
}
private boolean isConnectionInPool(String driverName) {
if (poolingDriverMap.containsKey(driverName)) return true;
return false;
}
private void createConnectionPool(String url, String propertyKey, Properties properties) {
ConnectionFactory connectionFactory =
new DriverManagerConnectionFactory(url, properties);
PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory(
connectionFactory, null);
ObjectPool connectionPool = new GenericObjectPool(poolableConnectionFactory);
poolableConnectionFactory.setPool(connectionPool);
PoolingDriver driver = new PoolingDriver();
driver.registerPool(propertyKey, connectionPool);
poolingDriverMap.put(propertyKey, driver);
}
private Connection getConnectionFromPool(String url, String propertyKey, Properties properties)
throws SQLException {
if (!isConnectionInPool(propertyKey)) {
createConnectionPool(url, propertyKey, properties);
}
return DriverManager.getConnection(DBCP_STRING + propertyKey);
}
public Connection getConnection(String propertyKey, String user)
throws ClassNotFoundException, SQLException, InterpreterException {
Connection connection = null;
if (propertyKey == null || propertiesMap.get(propertyKey) == null) {
return null;
}
if (null == connection) {
final Properties properties = (Properties) propertiesMap.get(propertyKey).clone();
logger.info(properties.getProperty(DRIVER_KEY));
Class.forName(properties.getProperty(DRIVER_KEY));
final String url = properties.getProperty(URL_KEY);
if (StringUtils.isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
connection = DriverManager.getConnection(url, properties);
} else {
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
switch (authType) {
case KERBEROS:
if (user == null) {
connection = getConnectionFromPool(url, propertyKey, properties);
} else {
if ("hive".equalsIgnoreCase(propertyKey)) {
connection = getConnectionFromPool(url + ";hive.server2.proxy.user=" + user,
propertyKey, properties);
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user,
UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in createProxyUser", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
final String poolKey = propertyKey;
try {
connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return getConnectionFromPool(url, poolKey, properties);
}
});
} catch (Exception e) {
logger.error("Error in doAs", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
}
}
break;
default:
connection = getConnectionFromPool(url, propertyKey, properties);
}
}
}
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(connection));
return connection;
}
private void initStatementMap() {
for (Statement statement : paragraphIdStatementMap.values()) {
for (JDBCUserConfigurations configurations : jdbcUserConfigurationsMap.values()) {
try {
statement.close();
configurations.initStatementMap();
} catch (Exception e) {
logger.error("Error while closing paragraphIdStatementMap statement...", e);
}
}
paragraphIdStatementMap.clear();
}
private void initConnectionPoolMap() throws SQLException {
Iterator<String> it = poolingDriverMap.keySet().iterator();
while (it.hasNext()) {
String driverName = it.next();
poolingDriverMap.get(driverName).closePool(driverName);
it.remove();
private void initConnectionPoolMap() {
for (JDBCUserConfigurations configurations : jdbcUserConfigurationsMap.values()) {
try {
configurations.initConnectionPoolMap();
} catch (Exception e) {
logger.error("Error while closing initConnectionPoolMap...", e);
}
}
poolingDriverMap.clear();
}
private void saveStatement(String key, Statement statement) throws SQLException {
paragraphIdStatementMap.put(key, statement);
statement.setMaxRows(getMaxResult());
}
private void removeStatement(String key) {
paragraphIdStatementMap.remove(key);
}
@Override
@ -328,15 +224,182 @@ public class JDBCInterpreter extends Interpreter {
}
}
private String getEntityName(String replName) {
StringBuffer entityName = new StringBuffer();
entityName.append(INTERPRETER_NAME);
entityName.append(".");
entityName.append(replName);
return entityName.toString();
}
private String getJDBCDriverName(String user, String propertyKey) {
StringBuffer driverName = new StringBuffer();
driverName.append(DBCP_STRING);
driverName.append(propertyKey);
driverName.append(user);
return driverName.toString();
}
private boolean existAccountInBaseProperty() {
return property.containsKey(JDBC_DEFAULT_USER_KEY) &&
property.containsKey(JDBC_DEFAULT_PASSWORD_KEY);
}
private UsernamePassword getUsernamePassword(InterpreterContext interpreterContext,
String replName) {
UserCredentials uc = interpreterContext.getAuthenticationInfo().getUserCredentials();
if (uc != null) {
return uc.getUsernamePassword(replName);
}
return null;
}
public JDBCUserConfigurations getJDBCConfiguration(String user) {
JDBCUserConfigurations jdbcUserConfigurations =
jdbcUserConfigurationsMap.get(user);
if (jdbcUserConfigurations == null) {
jdbcUserConfigurations = new JDBCUserConfigurations();
jdbcUserConfigurationsMap.put(user, jdbcUserConfigurations);
}
return jdbcUserConfigurations;
}
private void closeDBPool(String user, String propertyKey) throws SQLException {
PoolingDriver poolingDriver = getJDBCConfiguration(user).removeDBDriverPool(propertyKey);
if (poolingDriver != null) {
poolingDriver.closePool(propertyKey + user);
}
}
private void setUserProperty(String propertyKey, InterpreterContext interpreterContext)
throws SQLException {
String user = interpreterContext.getAuthenticationInfo().getUser();
JDBCUserConfigurations jdbcUserConfigurations =
getJDBCConfiguration(user);
jdbcUserConfigurations.setPropertyMap(propertyKey, basePropretiesMap.get(propertyKey));
if (existAccountInBaseProperty()) {
return;
}
jdbcUserConfigurations.cleanUserProperty(propertyKey);
UsernamePassword usernamePassword = getUsernamePassword(interpreterContext,
getEntityName(interpreterContext.getReplName()));
if (usernamePassword != null) {
jdbcUserConfigurations.setUserProperty(propertyKey, usernamePassword);
} else {
closeDBPool(user, propertyKey);
}
}
private void createConnectionPool(String url, String user, String propertyKey,
Properties properties) throws SQLException, ClassNotFoundException {
ConnectionFactory connectionFactory =
new DriverManagerConnectionFactory(url, properties);
PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory(
connectionFactory, null);
ObjectPool connectionPool = new GenericObjectPool(poolableConnectionFactory);
poolableConnectionFactory.setPool(connectionPool);
Class.forName(properties.getProperty(DRIVER_KEY));
PoolingDriver driver = new PoolingDriver();
driver.registerPool(propertyKey + user, connectionPool);
getJDBCConfiguration(user).saveDBDriverPool(propertyKey, driver);
}
private Connection getConnectionFromPool(String url, String user, String propertyKey,
Properties properties) throws SQLException, ClassNotFoundException {
String jdbcDriver = getJDBCDriverName(user, propertyKey);
if (!getJDBCConfiguration(user).isConnectionInDBDriverPool(propertyKey)) {
createConnectionPool(url, user, propertyKey, properties);
}
return DriverManager.getConnection(jdbcDriver);
}
public Connection getConnection(String propertyKey, InterpreterContext interpreterContext)
throws ClassNotFoundException, SQLException, InterpreterException {
final String user = interpreterContext.getAuthenticationInfo().getUser();
Connection connection;
if (propertyKey == null || basePropretiesMap.get(propertyKey) == null) {
return null;
}
JDBCUserConfigurations jdbcUserConfigurations = getJDBCConfiguration(user);
setUserProperty(propertyKey, interpreterContext);
final Properties properties = jdbcUserConfigurations.getPropertyMap(propertyKey);
final String url = properties.getProperty(URL_KEY);
if (StringUtils.isEmpty(property.getProperty("zeppelin.jdbc.auth.type"))) {
connection = getConnectionFromPool(url, user, propertyKey, properties);
} else {
UserGroupInformation.AuthenticationMethod authType = JDBCSecurityImpl.getAuthtype(property);
switch (authType) {
case KERBEROS:
if (user == null) {
connection = getConnectionFromPool(url, user, propertyKey, properties);
} else {
if ("hive".equalsIgnoreCase(propertyKey)) {
connection = getConnectionFromPool(url + ";hive.server2.proxy.user=" + user,
user, propertyKey, properties);
} else {
UserGroupInformation ugi = null;
try {
ugi = UserGroupInformation.createProxyUser(user,
UserGroupInformation.getCurrentUser());
} catch (Exception e) {
logger.error("Error in createProxyUser", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
final String poolKey = propertyKey;
try {
connection = ugi.doAs(new PrivilegedExceptionAction<Connection>() {
@Override
public Connection run() throws Exception {
return getConnectionFromPool(url, user, poolKey, properties);
}
});
} catch (Exception e) {
logger.error("Error in doAs", e);
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(e.getMessage()).append("\n");
stringBuilder.append(e.getCause());
throw new InterpreterException(stringBuilder.toString());
}
}
}
break;
default:
connection = getConnectionFromPool(url, user, propertyKey, properties);
}
}
propertyKeySqlCompleterMap.put(propertyKey, createSqlCompleter(connection));
return connection;
}
private InterpreterResult executeSql(String propertyKey, String sql,
InterpreterContext interpreterContext) {
String paragraphId = interpreterContext.getParagraphId();
Connection connection;
Statement statement;
ResultSet resultSet = null;
String paragraphId = interpreterContext.getParagraphId();
String user = interpreterContext.getAuthenticationInfo().getUser();
try {
connection = getConnection(propertyKey, interpreterContext.getAuthenticationInfo().getUser());
connection = getConnection(propertyKey, interpreterContext);
if (connection == null) {
return new InterpreterResult(Code.ERROR, "Prefix not found.");
}
@ -357,8 +420,7 @@ public class JDBCInterpreter extends Interpreter {
}
try {
saveStatement(paragraphId +
interpreterContext.getAuthenticationInfo().getUser(), statement);
getJDBCConfiguration(user).saveStatement(paragraphId, statement);
boolean isResultSetAvailable = statement.execute(sql);
@ -416,8 +478,7 @@ public class JDBCInterpreter extends Interpreter {
connection.close();
} catch (SQLException e) { /*ignored*/ }
}
removeStatement(paragraphId +
interpreterContext.getAuthenticationInfo().getUser());
getJDBCConfiguration(user).removeStatement(paragraphId);
}
return new InterpreterResult(Code.SUCCESS, msg.toString());
@ -427,6 +488,13 @@ public class JDBCInterpreter extends Interpreter {
PrintStream ps = new PrintStream(baos);
e.printStackTrace(ps);
String errorMsg = new String(baos.toByteArray(), StandardCharsets.UTF_8);
try {
closeDBPool(user, propertyKey);
} catch (SQLException e1) {
e1.printStackTrace();
}
return new InterpreterResult(Code.ERROR, errorMsg);
}
}
@ -458,12 +526,12 @@ public class JDBCInterpreter extends Interpreter {
@Override
public void cancel(InterpreterContext context) {
logger.info("Cancel current query statement.");
String paragraphId = context.getParagraphId();
JDBCUserConfigurations jdbcUserConfigurations =
getJDBCConfiguration(context.getAuthenticationInfo().getUser());
try {
paragraphIdStatementMap.get(paragraphId + context.getAuthenticationInfo().getUser()).cancel();
jdbcUserConfigurations.cancelStatement(paragraphId);
} catch (SQLException e) {
logger.error("Error while cancelling...", e);
}
@ -520,7 +588,7 @@ public class JDBCInterpreter extends Interpreter {
public int getMaxResult() {
return Integer.valueOf(
propertiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT));
basePropretiesMap.get(COMMON_KEY).getProperty(MAX_LINE_KEY, MAX_LINE_DEFAULT));
}
boolean isConcurrentExecution() {

100
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCUserConfigurations.java Normal file
View file

@ -0,0 +1,100 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
* agreements. See the NOTICE file distributed with this work for additional information regarding
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License. You may obtain a
* copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
* or implied. See the License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.zeppelin.jdbc;
import org.apache.commons.dbcp2.PoolingDriver;
import org.apache.zeppelin.user.UsernamePassword;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
/**
* UserConfigurations for JDBC impersonation.
*/
public class JDBCUserConfigurations {
private final Map<String, Statement> paragraphIdStatementMap;
private final Map<String, PoolingDriver> poolingDriverMap;
private final HashMap<String, Properties> propertiesMap;
public JDBCUserConfigurations() {
paragraphIdStatementMap = new HashMap<>();
poolingDriverMap = new HashMap<>();
propertiesMap = new HashMap<>();
}
public void initStatementMap() throws SQLException {
for (Statement statement : paragraphIdStatementMap.values()) {
statement.close();
}
paragraphIdStatementMap.clear();
}
public void initConnectionPoolMap() throws SQLException {
Iterator<String> it = poolingDriverMap.keySet().iterator();
while (it.hasNext()) {
String driverName = it.next();
poolingDriverMap.get(driverName).closePool(driverName);
it.remove();
}
poolingDriverMap.clear();
}
public void setPropertyMap(String key, Properties properties) {
Properties p = (Properties) properties.clone();
propertiesMap.put(key, p);
}
public Properties getPropertyMap(String key) {
return propertiesMap.get(key);
}
public void cleanUserProperty(String propertyKey) {
propertiesMap.get(propertyKey).remove("user");
propertiesMap.get(propertyKey).remove("password");
}
public void setUserProperty(String propertyKey, UsernamePassword usernamePassword) {
propertiesMap.get(propertyKey).setProperty("user", usernamePassword.getUsername());
propertiesMap.get(propertyKey).setProperty("password", usernamePassword.getPassword());
}
public void saveStatement(String key, Statement statement) throws SQLException {
paragraphIdStatementMap.put(key, statement);
}
public void cancelStatement(String key) throws SQLException {
paragraphIdStatementMap.get(key).cancel();
}
public void removeStatement(String key) {
paragraphIdStatementMap.remove(key);
}
public void saveDBDriverPool(String key, PoolingDriver driver) throws SQLException {
poolingDriverMap.put(key, driver);
}
public PoolingDriver removeDBDriverPool(String key) throws SQLException {
return poolingDriverMap.remove(key);
}
public boolean isConnectionInDBDriverPool(String key) {
return poolingDriverMap.containsKey(key);
}
}

104
jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java
View file

@ -16,20 +16,20 @@ package org.apache.zeppelin.jdbc;
import static java.lang.String.format;
import static org.apache.zeppelin.interpreter.Interpreter.logger;
import static org.junit.Assert.assertEquals;
import static org.apache.zeppelin.interpreter.Interpreter.register;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_KEY;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_DRIVER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_PASSWORD;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_USER;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.DEFAULT_URL;
import static org.apache.zeppelin.jdbc.JDBCInterpreter.COMMON_MAX_LINE;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.*;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;
@ -41,6 +41,9 @@ import org.apache.zeppelin.scheduler.FIFOScheduler;
import org.apache.zeppelin.scheduler.ParallelScheduler;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
import org.apache.zeppelin.user.UserCredentials;
import org.apache.zeppelin.user.UsernamePassword;
import org.junit.Before;
import org.junit.Test;
@ -75,7 +78,6 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
@Before
public void setUp() throws Exception {
Class.forName("org.h2.Driver");
Connection connection = DriverManager.getConnection(getJdbcConnection());
Statement statement = connection.createStatement();
@ -86,7 +88,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
PreparedStatement insertStatement = connection.prepareStatement("insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);");
insertStatement.setString(1, null);
insertStatement.execute();
interpreterContext = new InterpreterContext("", "1", "", "", new AuthenticationInfo(), null, null, null, null,
interpreterContext = new InterpreterContext("", "1", null, "", "", new AuthenticationInfo(), null, null, null, null,
null, null);
}
@ -251,7 +253,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
jdbcInterpreter.interpret("", interpreterContext);
List<InterpreterCompletion> completionList = jdbcInterpreter.completion("SEL", 0);
InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("SELECT", "SELECT");
assertEquals(2, completionList.size());
@ -259,4 +261,92 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
assertEquals(0, jdbcInterpreter.completion("SEL", 100).size());
}
}
private Properties getDBProperty(String dbUser, String dbPassowrd) throws IOException {
Properties properties = new Properties();
properties.setProperty("common.max_count", "1000");
properties.setProperty("common.max_retry", "3");
properties.setProperty("default.driver", "org.h2.Driver");
properties.setProperty("default.url", getJdbcConnection());
if (dbUser != null) {
properties.setProperty("default.user", dbUser);
}
if (dbPassowrd != null) {
properties.setProperty("default.password", dbPassowrd);
}
return properties;
}
private AuthenticationInfo getUserAuth(String user, String entityName, String dbUser, String dbPassword){
UserCredentials userCredentials = new UserCredentials();
if (entityName != null && dbUser != null && dbPassword != null) {
UsernamePassword up = new UsernamePassword(dbUser, dbPassword);
userCredentials.putUsernamePassword(entityName, up);
}
AuthenticationInfo authInfo = new AuthenticationInfo();
authInfo.setUserCredentials(userCredentials);
authInfo.setUser(user);
return authInfo;
}
@Test
public void testMultiTenant() throws SQLException, IOException {
/**
* assume that the database user is 'dbuser' and password is 'dbpassword'
* 'jdbc1' interpreter has user('dbuser')/password('dbpassword') property
* 'jdbc2' interpreter doesn't have user/password property
* 'user1' doesn't have Credential information.
* 'user2' has 'jdbc2' Credential information that is same with database account.
*/
JDBCInterpreter jdbc1 = new JDBCInterpreter(getDBProperty("dbuser", "dbpassword"));
JDBCInterpreter jdbc2 = new JDBCInterpreter(getDBProperty(null, null));
AuthenticationInfo user1Credential = getUserAuth("user1", null, null, null);
AuthenticationInfo user2Credential = getUserAuth("user2", "jdbc.jdbc2", "dbuser", "dbpassword");
// user1 runs jdbc1
jdbc1.open();
InterpreterContext ctx1 = new InterpreterContext("", "1", "jdbc.jdbc1", "", "", user1Credential,
null, null, null, null, null, null);
jdbc1.interpret("", ctx1);
JDBCUserConfigurations user1JDBC1Conf = jdbc1.getJDBCConfiguration("user1");
assertEquals("dbuser", user1JDBC1Conf.getPropertyMap("default").get("user"));
assertEquals("dbpassword", user1JDBC1Conf.getPropertyMap("default").get("password"));
jdbc1.close();
// user1 runs jdbc2
jdbc2.open();
InterpreterContext ctx2 = new InterpreterContext("", "1", "jdbc.jdbc2", "", "", user1Credential,
null, null, null, null, null, null);
jdbc2.interpret("", ctx2);
JDBCUserConfigurations user1JDBC2Conf = jdbc2.getJDBCConfiguration("user1");
assertNull(user1JDBC2Conf.getPropertyMap("default").get("user"));
assertNull(user1JDBC2Conf.getPropertyMap("default").get("password"));
jdbc2.close();
// user2 runs jdbc1
jdbc1.open();
InterpreterContext ctx3 = new InterpreterContext("", "1", "jdbc.jdbc1", "", "", user2Credential,
null, null, null, null, null, null);
jdbc1.interpret("", ctx3);
JDBCUserConfigurations user2JDBC1Conf = jdbc1.getJDBCConfiguration("user2");
assertEquals("dbuser", user2JDBC1Conf.getPropertyMap("default").get("user"));
assertEquals("dbpassword", user2JDBC1Conf.getPropertyMap("default").get("password"));
jdbc1.close();
// user2 runs jdbc2
jdbc2.open();
InterpreterContext ctx4 = new InterpreterContext("", "1", "jdbc.jdbc2", "", "", user2Credential,
null, null, null, null, null, null);
jdbc2.interpret("", ctx4);
JDBCUserConfigurations user2JDBC2Conf = jdbc2.getJDBCConfiguration("user2");
assertNull(user2JDBC2Conf.getPropertyMap("default").get("user"));
assertNull(user2JDBC2Conf.getPropertyMap("default").get("password"));
jdbc2.close();
}
}

16
lens/src/main/java/org/apache/zeppelin/lens/LensInterpreter.java
View file

@ -88,21 +88,7 @@ public class LensInterpreter extends Interpreter {
private LensClient m_lensClient;
static {
Interpreter.register(
"lens",
"lens",
LensInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add(ZEPPELIN_LENS_RUN_CONCURRENT_SESSION, "true", "Run concurrent Lens Sessions")
.add(ZEPPELIN_LENS_CONCURRENT_SESSIONS, "10",
"If concurrency is true then how many threads?")
.add(ZEPPELIN_MAX_ROWS, "1000", "max number of rows to display")
.add(LENS_SERVER_URL, "http://<hostname>:<port>/lensapi", "The URL for Lens Server")
.add(LENS_CLIENT_DBNAME, "default", "The database schema name")
.add(LENS_PERSIST_RESULTSET, "false", "Apache Lens to persist result in HDFS?")
.add(LENS_SESSION_CLUSTER_USER, "default", "Hadoop cluster username").build());
}
public LensInterpreter(Properties property) {
super(property);

51
lens/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,51 @@
[
{
"group": "lens",
"name": "lens",
"className": "org.apache.zeppelin.lens.LensInterpreter",
"properties": {
"zeppelin.lens.run.concurrent": {
"envName": null,
"propertyName": "zeppelin.lens.run.concurrent",
"defaultValue": "true",
"description": "Run concurrent Lens Sessions"
},
"zeppelin.lens.maxThreads": {
"envName": null,
"propertyName": "zeppelin.lens.maxThreads",
"defaultValue": "10",
"description": "If concurrency is true then how many threads?"
},
"zeppelin.lens.maxResults": {
"envName": null,
"propertyName": "zeppelin.lens.maxResults",
"defaultValue": "1000",
"description": "max number of rows to display"
},
"lens.server.base.url": {
"envName": null,
"propertyName": "lens.server.base.url",
"defaultValue": "http://<hostname>:<port>/lensapi",
"description": "The URL for Lens Server"
},
"lens.client.dbname": {
"envName": null,
"propertyName": "lens.client.dbname",
"defaultValue": "default",
"description": "The database schema name"
},
"lens.query.enable.persistent.resultset": {
"envName": null,
"propertyName": "lens.query.enable.persistent.resultset",
"defaultValue": "false",
"description": "Apache Lens to persist result in HDFS?"
},
"lens.session.cluster.user": {
"envName": null,
"propertyName": "lens.session.cluster.user",
"defaultValue": "default",
"description": "Hadoop cluster username"
}
}
}
]

4
livy/src/test/java/org/apache/zeppelin/livy/LivyIntegrationTest.java
View file

@ -81,7 +81,7 @@ public class LivyIntegrationTest {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "title",
InterpreterContext context = new InterpreterContext("noteId", "paragraphId", null, "title",
"text", authInfo, null, null, null, null, null, output);
sparkInterpreter.open();
InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
@ -175,7 +175,7 @@ public class LivyIntegrationTest {
AuthenticationInfo authInfo = new AuthenticationInfo("user1");
MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
InterpreterOutput output = new InterpreterOutput(outputListener);
InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "title",
InterpreterContext context = new InterpreterContext("noteId", "paragraphId", null, "title",
"text", authInfo, null, null, null, null, null, output);
pysparkInterpreter.open();
InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);

4
notebook/2A94M5J1Z/note.json
View file

@ -420,7 +420,7 @@
"progressUpdateIntervalMs": 500
}
],
"name": "Zeppelin Tutorial",
"name": "Zeppelin Tutorial/Basic Features (Spark)",
"id": "2A94M5J1Z",
"angularObjects": {
"2C2HP46V7:shared_process": [],
@ -435,4 +435,4 @@
"looknfeel": "default"
},
"info": {}
}
}

4
notebook/2BWJFTXKJ/note.json
View file

@ -1037,7 +1037,7 @@
"progressUpdateIntervalMs": 500
}
],
"name": "R Tutorial",
"name": "Zeppelin Tutorial/R (SparkR)",
"id": "2BWJFTXKJ",
"lastReplName": {
"value": ""
@ -1066,4 +1066,4 @@
"looknfeel": "default"
},
"info": {}
}
}

4
notebook/2BYEZ5EVK/note.json
View file

@ -743,7 +743,7 @@
"progressUpdateIntervalMs": 500
}
],
"name": "Mahout Tutorial",
"name": "Zeppelin Tutorial/Using Mahout",
"id": "2BYEZ5EVK",
"angularObjects": {
"2BWWSVNY7:shared_process": [],
@ -769,4 +769,4 @@
},
"config": {},
"info": {}
}
}

4
notebook/2C2AUG798/note.json
View file

@ -669,7 +669,7 @@
"progressUpdateIntervalMs": 500
}
],
"name": "Zeppelin Tutorial: Python - matplotlib basic",
"name": "Zeppelin Tutorial/Matplotlib (Python • PySpark)",
"id": "2C2AUG798",
"angularObjects": {
"2BWWUCUVW:shared_process": [],
@ -693,4 +693,4 @@
},
"config": {},
"info": {}
}
}

2
pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
View file

@ -47,7 +47,7 @@ public class PigInterpreterTest {
properties.put("zeppelin.pig.execType", "local");
pigInterpreter = new PigInterpreter(properties);
pigInterpreter.open();
context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null,
context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null, null,
null, null);
}

2
pig/src/test/java/org/apache/zeppelin/pig/PigQueryInterpreterTest.java
View file

@ -65,7 +65,7 @@ public class PigQueryInterpreterTest {
pigInterpreter.open();
pigQueryInterpreter.open();
context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null,
context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null, null, null, null,
null, null);
}

40
pom.xml
View file

@ -385,6 +385,11 @@
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>xml-maven-plugin</artifactId>
</plugin>
<!--TODO(alex): make part of the build and reconcile conflicts
<plugin>
<groupId>com.ning.maven.plugins</groupId>
@ -422,11 +427,8 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration combine.children="append">
<configuration>
<argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
<encoding>UTF-8</encoding>
<inputEncoding>UTF-8</inputEncoding>
<outputEncoding>UTF-8</outputEncoding>
</configuration>
<!-- <excludes> <exclude>**/itest/**</exclude> </excludes> <executions>
<execution> <id>surefire-itest</id> <phase>integration-test</phase> <goals>
@ -437,7 +439,7 @@
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4</version>
<version>3.0.0</version>
</plugin>
<plugin>
@ -469,6 +471,33 @@
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>xml-maven-plugin</artifactId>
<version>1.0.1</version>
<executions>
<execution>
<phase>validate</phase>
<goals>
<goal>validate</goal>
</goals>
</execution>
</executions>
<configuration>
<validationSets>
<validationSet>
<dir>${project.basedir}</dir>
<includes>
<include>
pom.xml
</include>
</includes>
<systemId>_tools/maven-4.0.0.xsd</systemId>
</validationSet>
</validationSets>
</configuration>
</plugin>
<!--This plugin's configuration is used to store Eclipse m2e settings
only. It has no influence on the Maven build itself. -->
<plugin>
@ -750,6 +779,7 @@
<exclude>**/null/**</exclude>
<exclude>**/notebook/**</exclude>
<exclude>_tools/site/css/*</exclude>
<exclude>_tools/maven-4.0.0.xsd</exclude>
<exclude>**/README.md</exclude>
<exclude>DEPENDENCIES</exclude>
<exclude>DEPLOY.md</exclude>

208
python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java Normal file
View file

@ -0,0 +1,208 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Conda support
*/
public class PythonCondaInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(PythonCondaInterpreter.class);
Pattern condaEnvListPattern = Pattern.compile("([^\\s]*)[\\s*]*\\s(.*)");
Pattern listPattern = Pattern.compile("env\\s*list\\s?");
Pattern activatePattern = Pattern.compile("activate\\s*(.*)");
Pattern deactivatePattern = Pattern.compile("deactivate");
Pattern helpPattern = Pattern.compile("help");
public PythonCondaInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
InterpreterOutput out = context.out;
Matcher listMatcher = listPattern.matcher(st);
Matcher activateMatcher = activatePattern.matcher(st);
Matcher deactivateMatcher = deactivatePattern.matcher(st);
Matcher helpMatcher = helpPattern.matcher(st);
if (st == null || st.isEmpty() || listMatcher.matches()) {
listEnv(out);
return new InterpreterResult(InterpreterResult.Code.SUCCESS);
} else if (activateMatcher.matches()) {
String envName = activateMatcher.group(1);
setPythonCommand("conda run -n " + envName + " \"python -iu\"");
restartPythonProcess();
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "\"" + envName + "\" activated");
} else if (deactivateMatcher.matches()) {
setPythonCommand(null);
restartPythonProcess();
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "Deactivated");
} else if (helpMatcher.matches()) {
printUsage(out);
return new InterpreterResult(InterpreterResult.Code.SUCCESS);
} else {
return new InterpreterResult(InterpreterResult.Code.ERROR, "Not supported command: " + st);
}
}
public void setPythonCommand(String cmd) {
PythonInterpreter python = getPythonInterpreter();
python.setPythonCommand(cmd);
}
private void restartPythonProcess() {
PythonInterpreter python = getPythonInterpreter();
python.close();
python.open();
}
protected PythonInterpreter getPythonInterpreter() {
LazyOpenInterpreter lazy = null;
PythonInterpreter python = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
while (p instanceof WrappedInterpreter) {
if (p instanceof LazyOpenInterpreter) {
lazy = (LazyOpenInterpreter) p;
}
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
python = (PythonInterpreter) p;
if (lazy != null) {
lazy.open();
}
return python;
}
private void listEnv(InterpreterOutput out) {
StringBuilder sb = createStringBuilder();
try {
int exit = runCommand(sb, "conda", "env", "list");
if (exit == 0) {
out.setType(InterpreterResult.Type.HTML);
out.write("<h4>Conda environments</h4>\n");
// start table
out.write("<div style=\"display:table\">\n");
String[] lines = sb.toString().split("\n");
for (String s : lines) {
if (s == null || s.isEmpty() || s.startsWith("#")) {
continue;
}
Matcher match = condaEnvListPattern.matcher(s);
if (!match.matches()) {
continue;
}
out.write(String.format("<div style=\"display:table-row\">" +
"<div style=\"display:table-cell;width:150px\">%s</div>" +
"<div style=\"display:table-cell;\">%s</div>" +
"</div>\n",
match.group(1), match.group(2)));
}
// end table
out.write("</div><br />\n");
out.write("<small><code>%python.conda help</code> for the usage</small>\n");
} else {
out.write("Failed to run 'conda' " + exit + "\n");
}
} catch (IOException | InterruptedException e) {
throw new InterpreterException(e);
}
}
private void printUsage(InterpreterOutput out) {
try {
out.setType(InterpreterResult.Type.HTML);
out.writeResource("output_templates/conda_usage.html");
} catch (IOException e) {
logger.error("Can't print usage", e);
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NONE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
/**
* Use python interpreter's scheduler.
* To make sure %python.conda paragraph and %python paragraph runs sequentially
*/
@Override
public Scheduler getScheduler() {
PythonInterpreter pythonInterpreter = getPythonInterpreter();
if (pythonInterpreter != null) {
return pythonInterpreter.getScheduler();
} else {
return null;
}
}
protected int runCommand(StringBuilder sb, String ... command)
throws IOException, InterruptedException {
ProcessBuilder builder = new ProcessBuilder(command);
builder.redirectErrorStream(true);
Process process = builder.start();
InputStream stdout = process.getInputStream();
BufferedReader br = new BufferedReader(new InputStreamReader(stdout));
String line;
while ((line = br.readLine()) != null) {
sb.append(line);
sb.append("\n");
}
int r = process.waitFor(); // Let the process finish.
return r;
}
protected StringBuilder createStringBuilder() {
return new StringBuilder();
}
}

175
python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java Normal file
View file

@ -0,0 +1,175 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Helps run python interpreter on a docker container
*/
public class PythonDockerInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(PythonDockerInterpreter.class);
Pattern activatePattern = Pattern.compile("activate\\s*(.*)");
Pattern deactivatePattern = Pattern.compile("deactivate");
Pattern helpPattern = Pattern.compile("help");
public PythonDockerInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
InterpreterOutput out = context.out;
Matcher activateMatcher = activatePattern.matcher(st);
Matcher deactivateMatcher = deactivatePattern.matcher(st);
Matcher helpMatcher = helpPattern.matcher(st);
if (st == null || st.isEmpty() || helpMatcher.matches()) {
printUsage(out);
return new InterpreterResult(InterpreterResult.Code.SUCCESS);
} else if (activateMatcher.matches()) {
String image = activateMatcher.group(1);
pull(out, image);
setPythonCommand("docker run -i --rm " + image + " python -iu");
restartPythonProcess();
out.clear();
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "\"" + image + "\" activated");
} else if (deactivateMatcher.matches()) {
setPythonCommand(null);
restartPythonProcess();
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "Deactivated");
} else {
return new InterpreterResult(InterpreterResult.Code.ERROR, "Not supported command: " + st);
}
}
public void setPythonCommand(String cmd) {
PythonInterpreter python = getPythonInterpreter();
python.setPythonCommand(cmd);
}
private void printUsage(InterpreterOutput out) {
try {
out.setType(InterpreterResult.Type.HTML);
out.writeResource("output_templates/docker_usage.html");
} catch (IOException e) {
logger.error("Can't print usage", e);
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NONE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
/**
* Use python interpreter's scheduler.
* To make sure %python.docker paragraph and %python paragraph runs sequentially
*/
@Override
public Scheduler getScheduler() {
PythonInterpreter pythonInterpreter = getPythonInterpreter();
if (pythonInterpreter != null) {
return pythonInterpreter.getScheduler();
} else {
return null;
}
}
private void restartPythonProcess() {
PythonInterpreter python = getPythonInterpreter();
python.close();
python.open();
}
protected PythonInterpreter getPythonInterpreter() {
LazyOpenInterpreter lazy = null;
PythonInterpreter python = null;
Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
while (p instanceof WrappedInterpreter) {
if (p instanceof LazyOpenInterpreter) {
lazy = (LazyOpenInterpreter) p;
}
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
python = (PythonInterpreter) p;
if (lazy != null) {
lazy.open();
}
return python;
}
public boolean pull(InterpreterOutput out, String image) {
int exit = 0;
try {
exit = runCommand(out, "docker", "pull", image);
} catch (IOException | InterruptedException e) {
logger.error(e.getMessage(), e);
throw new InterpreterException(e);
}
return exit == 0;
}
protected int runCommand(InterpreterOutput out, String... command)
throws IOException, InterruptedException {
ProcessBuilder builder = new ProcessBuilder(command);
builder.redirectErrorStream(true);
Process process = builder.start();
InputStream stdout = process.getInputStream();
BufferedReader br = new BufferedReader(new InputStreamReader(stdout));
String line;
while ((line = br.readLine()) != null) {
out.write(line + "\n");
}
int r = process.waitFor(); // Let the process finish.
return r;
}
}

22
python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
View file

@ -28,12 +28,9 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Scheduler;
@ -63,6 +60,7 @@ public class PythonInterpreter extends Interpreter {
private int maxResult;
PythonProcess process = null;
private String pythonCommand = null;
public PythonInterpreter(Properties property) {
super(property);
@ -123,6 +121,7 @@ public class PythonInterpreter extends Interpreter {
try {
if (process != null) {
process.close();
process = null;
}
if (gatewayServer != null) {
gatewayServer.shutdown();
@ -201,12 +200,24 @@ public class PythonInterpreter extends Interpreter {
public PythonProcess getPythonProcess() {
if (process == null) {
return new PythonProcess(getProperty(ZEPPELIN_PYTHON));
String binPath = getProperty(ZEPPELIN_PYTHON);
if (pythonCommand != null) {
binPath = pythonCommand;
}
return new PythonProcess(binPath);
} else {
return process;
}
}
public void setPythonCommand(String cmd) {
pythonCommand = cmd;
}
public String getPythonCommand() {
return pythonCommand;
}
private Job getRunningJob(String paragraphId) {
Job foundJob = null;
Collection<Job> jobsRunning = getScheduler().getJobsRunning();
@ -281,5 +292,4 @@ public class PythonInterpreter extends Interpreter {
public int getMaxResult() {
return maxResult;
}
}

18
python/src/main/java/org/apache/zeppelin/python/PythonProcess.java
View file

@ -49,7 +49,23 @@ public class PythonProcess {
}
public void open() throws IOException {
ProcessBuilder builder = new ProcessBuilder(binPath, "-iu");
ProcessBuilder builder;
boolean hasParams = binPath.split(" ").length > 1;
if (System.getProperty("os.name").toLowerCase().contains("windows")) {
if (hasParams) {
builder = new ProcessBuilder(binPath.split(" "));
} else {
builder = new ProcessBuilder(binPath, "-iu");
}
} else {
String cmd;
if (hasParams) {
cmd = binPath;
} else {
cmd = binPath + " -iu";
}
builder = new ProcessBuilder("bash", "-c", cmd);
}
builder.redirectErrorStream(true);
process = builder.start();

11
python/src/main/resources/interpreter-setting.json
View file

@ -32,5 +32,16 @@
"language": "sql",
"editOnDblClick": false
}
},
{
"group": "python",
"name": "conda",
"className": "org.apache.zeppelin.python.PythonCondaInterpreter",
"properties": {
},
"editor":{
"language": "sh",
"editOnDblClick": false
}
}
]

27
python/src/main/resources/output_templates/conda_usage.html Normal file
View file

@ -0,0 +1,27 @@
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<h4>Usage</h4>
<div>
Activate an environment (python interpreter will be restarted)
<pre>%python.conda activate [ENV NAME]</pre>
</div>
<div>
Deactivate
<pre>%python.conda deactivate</pre>
</div>
<div>
List the Conda environments
<pre>%python.conda</pre>
</div>

27
python/src/main/resources/output_templates/docker_usage.html Normal file
View file

@ -0,0 +1,27 @@
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<h4>Usage</h4>
<div>
Activate an docker environment (python interpreter will be restarted)
<pre>%python.docker activate [Repository]
%python.docker activate [Repository:Tag]
%python.docker activate [Image Id]</pre>
</div>
<div>
Deactivate
<pre>%python.docker deactivate</pre>
</div>
<div>
Example
<pre># Run python interpreter with latest tensorflow image
%python.docker activate gcr.io/tensorflow/tensorflow:latest</pre>
</div>

117
python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java Normal file
View file

@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.*;
import static org.mockito.Mockito.*;
public class PythonCondaInterpreterTest implements InterpreterOutputListener {
private PythonCondaInterpreter conda;
private PythonInterpreter python;
@Before
public void setUp() {
conda = spy(new PythonCondaInterpreter(new Properties()));
python = mock(PythonInterpreter.class);
InterpreterGroup group = new InterpreterGroup();
group.put("note", Arrays.asList(python, conda));
python.setInterpreterGroup(group);
conda.setInterpreterGroup(group);
doReturn(python).when(conda).getPythonInterpreter();
}
@Test
public void testListEnv() throws IOException, InterruptedException {
InterpreterContext context = getInterpreterContext();
StringBuilder sb = new StringBuilder();
sb.append("#comment\n\nenv1 * /path1\nenv2\t/path2\n");
doReturn(sb).when(conda).createStringBuilder();
doReturn(0).when(conda)
.runCommand(any(StringBuilder.class), anyString(), anyString(), anyString());
// list available env
InterpreterResult result = conda.interpret("", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
String out = new String(context.out.toByteArray());
assertTrue(out.contains(">env1<"));
assertTrue(out.contains(">/path1<"));
assertTrue(out.contains(">env2<"));
assertTrue(out.contains(">/path2<"));
}
@Test
public void testActivateEnv() {
InterpreterContext context = getInterpreterContext();
conda.interpret("activate env", context);
verify(python, times(1)).open();
verify(python, times(1)).close();
verify(python).setPythonCommand("conda run -n env \"python -iu\"");
}
@Test
public void testDeactivate() {
InterpreterContext context = getInterpreterContext();
conda.interpret("deactivate", context);
verify(python, times(1)).open();
verify(python, times(1)).close();
verify(python).setPythonCommand(null);
}
private InterpreterContext getInterpreterContext() {
return new InterpreterContext(
"noteId",
"paragraphId",
null,
"paragraphTitle",
"paragraphText",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
null,
null,
null,
new InterpreterOutput(this));
}
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}

96
python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java Normal file
View file

@ -0,0 +1,96 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.python;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.*;
public class PythonDockerInterpreterTest implements InterpreterOutputListener {
private PythonDockerInterpreter docker;
private PythonInterpreter python;
@Before
public void setUp() {
docker = spy(new PythonDockerInterpreter(new Properties()));
python = mock(PythonInterpreter.class);
InterpreterGroup group = new InterpreterGroup();
group.put("note", Arrays.asList(python, docker));
python.setInterpreterGroup(group);
docker.setInterpreterGroup(group);
doReturn(true).when(docker).pull(any(InterpreterOutput.class), anyString());
doReturn(python).when(docker).getPythonInterpreter();
}
@Test
public void testActivateEnv() {
InterpreterContext context = getInterpreterContext();
docker.interpret("activate env", context);
verify(python, times(1)).open();
verify(python, times(1)).close();
verify(docker, times(1)).pull(any(InterpreterOutput.class), anyString());
verify(python).setPythonCommand("docker run -i --rm env python -iu");
}
@Test
public void testDeactivate() {
InterpreterContext context = getInterpreterContext();
docker.interpret("deactivate", context);
verify(python, times(1)).open();
verify(python, times(1)).close();
verify(python).setPythonCommand(null);
}
private InterpreterContext getInterpreterContext() {
return new InterpreterContext(
"noteId",
"paragraphId",
"paragraphTitle",
"paragraphText",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
null,
null,
null,
new InterpreterOutput(this));
}
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}

2
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java
View file

@ -76,7 +76,7 @@ public class PythonInterpreterMatplotlibTest {
interpreters.add(python);
intpGroup.put("note", interpreters);
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null), null,
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));

2
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java
View file

@ -78,7 +78,7 @@ public class PythonInterpreterPandasSqlTest {
intpGroup.put("note", Arrays.asList(python, sql));
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null), null,
new LinkedList<InterpreterContextRunner>(), new InterpreterOutput(null));

20
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
View file

@ -37,9 +37,13 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketAddress;
import java.util.LinkedList;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.Before;
import org.junit.Test;
@ -82,8 +86,15 @@ public class PythonInterpreterTest {
}
});
// python interpreter
pythonInterpreter = spy(new PythonInterpreter(getPythonTestProperties()));
// create interpreter group
InterpreterGroup group = new InterpreterGroup();
group.put("note", new LinkedList<Interpreter>());
group.get("note").add(pythonInterpreter);
pythonInterpreter.setInterpreterGroup(group);
when(pythonInterpreter.getPythonProcess()).thenReturn(mockPythonProcess);
when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("ImportError");
}
@ -220,15 +231,24 @@ public class PythonInterpreterTest {
@Test
public void checkMultiRowErrorFails() {
PythonInterpreter pythonInterpreter = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties()
);
// create interpreter group
InterpreterGroup group = new InterpreterGroup();
group.put("note", new LinkedList<Interpreter>());
group.get("note").add(pythonInterpreter);
pythonInterpreter.setInterpreterGroup(group);
pythonInterpreter.open();
String codeRaiseException = "raise Exception(\"test exception\")";
InterpreterResult ret = pythonInterpreter.interpret(codeRaiseException, null);
assertNotNull("Interpreter result for raise exception is Null", ret);
System.err.println("ret = '" + ret + "'");
assertEquals(InterpreterResult.Code.ERROR, ret.code());
assertTrue(ret.message().get(0).getData().length() > 0);

15
python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
View file

@ -21,9 +21,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.Test;
import java.util.Arrays;
/**
* Python interpreter unit test that user real Python
*
@ -46,6 +50,11 @@ public class PythonInterpreterWithPythonInstalledTest {
//given
PythonInterpreter realPython = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties());
// create interpreter group
InterpreterGroup group = new InterpreterGroup();
group.put("note", Arrays.asList((Interpreter) realPython));
realPython.setInterpreterGroup(group);
realPython.open();
//when
@ -65,6 +74,9 @@ public class PythonInterpreterWithPythonInstalledTest {
//given
PythonInterpreter realPython = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties());
InterpreterGroup group = new InterpreterGroup();
group.put("note", Arrays.asList((Interpreter) realPython));
realPython.setInterpreterGroup(group);
realPython.open();
//when
@ -84,6 +96,9 @@ public class PythonInterpreterWithPythonInstalledTest {
//given
PythonInterpreter realPython = new PythonInterpreter(
PythonInterpreterTest.getPythonTestProperties());
InterpreterGroup group = new InterpreterGroup();
group.put("note", Arrays.asList((Interpreter) realPython));
realPython.setInterpreterGroup(group);
realPython.open();
//when

2
scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java
View file

@ -64,7 +64,7 @@ public class ScaldingInterpreterTest {
}
InterpreterGroup intpGroup = new InterpreterGroup();
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null), null,
new LinkedList<InterpreterContextRunner>(), null);

2
scio/src/test/java/org/apache/zeppelin/scio/ScioInterpreterTest.java
View file

@ -40,7 +40,7 @@ public class ScioInterpreterTest {
private final String newline = "\n";
private InterpreterContext getNewContext() {
return new InterpreterContext("note", "id", "title", "text",
return new InterpreterContext("note", "id", null, "title", "text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),

4
shell/src/test/java/org/apache/zeppelin/shell/ShellInterpreterTest.java
View file

@ -47,7 +47,7 @@ public class ShellInterpreterTest {
@Test
public void test() {
shell.open();
InterpreterContext context = new InterpreterContext("", "1", "", "", null, null, null, null, null, null, null);
InterpreterContext context = new InterpreterContext("", "1", null, "", "", null, null, null, null, null, null, null);
InterpreterResult result = new InterpreterResult(Code.ERROR);
if (System.getProperty("os.name").startsWith("Windows")) {
result = shell.interpret("dir", context);
@ -64,7 +64,7 @@ public class ShellInterpreterTest {
@Test
public void testInvalidCommand(){
shell.open();
InterpreterContext context = new InterpreterContext("","1","","",null,null,null,null,null,null,null);
InterpreterContext context = new InterpreterContext("","1",null,"","",null,null,null,null,null,null,null);
InterpreterResult result = new InterpreterResult(Code.ERROR);
if (System.getProperty("os.name").startsWith("Windows")) {
result = shell.interpret("invalid_command\ndir",context);

8
spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java
View file

@ -42,8 +42,6 @@ import org.sonatype.aether.util.artifact.JavaScopes;
import org.sonatype.aether.util.filter.DependencyFilterUtils;
import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
import scala.Console;
/**
*
@ -66,8 +64,6 @@ public class SparkDependencyContext {
}
public Dependency load(String lib) {
Console.println("DepInterpreter(%dep) deprecated. "
+ "Load dependency through GUI interpreter menu instead.");
Dependency dep = new Dependency(lib);
if (dependencies.contains(dep)) {
@ -78,16 +74,12 @@ public class SparkDependencyContext {
}
public Repository addRepo(String name) {
Console.println("DepInterpreter(%dep) deprecated. "
+ "Add repository through GUI interpreter menu instead.");
Repository rep = new Repository(name);
repositories.add(rep);
return rep;
}
public void reset() {
Console.println("DepInterpreter(%dep) deprecated. "
+ "Remove dependencies and repositories through GUI interpreter menu instead.");
dependencies = new LinkedList<>();
repositories = new LinkedList<>();

2
spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java
View file

@ -64,7 +64,7 @@ public class DepInterpreterTest {
intpGroup.get("note").add(dep);
dep.setInterpreterGroup(intpGroup);
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,

2
spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java
View file

@ -94,7 +94,7 @@ public class PySparkInterpreterTest {
pySparkInterpreter.open();
}
context = new InterpreterContext("note", "id", "title", "text",
context = new InterpreterContext("note", "id", null, "title", "text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),

2
spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
View file

@ -92,7 +92,7 @@ public class SparkInterpreterTest {
repl.open();
}
context = new InterpreterContext("note", "id", "title", "text",
context = new InterpreterContext("note", "id", null, "title", "text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),

2
spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
View file

@ -75,7 +75,7 @@ public class SparkSqlInterpreterTest {
sql.setInterpreterGroup(intpGroup);
sql.open();
}
context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
new HashMap<String, Object>(), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("id"),

2
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/AbstractAngularElemTest.scala
View file

@ -33,7 +33,7 @@ trait AbstractAngularElemTest
override def beforeEach() {
val intpGroup = new InterpreterGroup()
val context = new InterpreterContext("note", "paragraph", "title", "text",
val context = new InterpreterContext("note", "paragraph", null, "title", "text",
new AuthenticationInfo(), new util.HashMap[String, Object](), new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,

2
zeppelin-display/src/test/scala/org/apache/zeppelin/display/angular/AbstractAngularModelTest.scala
View file

@ -29,7 +29,7 @@ trait AbstractAngularModelTest extends FlatSpec
with BeforeAndAfter with BeforeAndAfterEach with Eventually with Matchers {
override def beforeEach() {
val intpGroup = new InterpreterGroup()
val context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(),
val context = new InterpreterContext("note", "id", null, "title", "text", new AuthenticationInfo(),
new java.util.HashMap[String, Object](), new GUI(), new AngularObjectRegistry(
intpGroup.getId(), null),
null,

12
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
View file

@ -49,6 +49,7 @@ public class InterpreterContext {
}
private final String noteId;
private final String replName;
private final String paragraphTitle;
private final String paragraphId;
private final String paragraphText;
@ -63,6 +64,7 @@ public class InterpreterContext {
public InterpreterContext(String noteId,
String paragraphId,
String replName,
String paragraphTitle,
String paragraphText,
AuthenticationInfo authenticationInfo,
@ -75,6 +77,7 @@ public class InterpreterContext {
) {
this.noteId = noteId;
this.paragraphId = paragraphId;
this.replName = replName;
this.paragraphTitle = paragraphTitle;
this.paragraphText = paragraphText;
this.authenticationInfo = authenticationInfo;
@ -88,6 +91,7 @@ public class InterpreterContext {
public InterpreterContext(String noteId,
String paragraphId,
String replName,
String paragraphTitle,
String paragraphText,
AuthenticationInfo authenticationInfo,
@ -98,8 +102,8 @@ public class InterpreterContext {
List<InterpreterContextRunner> contextRunners,
InterpreterOutput output,
RemoteInterpreterEventClient eventClient) {
this(noteId, paragraphId, paragraphTitle, paragraphText, authenticationInfo, config, gui,
angularObjectRegistry, resourcePool, contextRunners, output);
this(noteId, paragraphId, replName, paragraphTitle, paragraphText, authenticationInfo,
config, gui, angularObjectRegistry, resourcePool, contextRunners, output);
this.client = new RemoteEventClient(eventClient);
}
@ -107,6 +111,10 @@ public class InterpreterContext {
return noteId;
}
public String getReplName() {
return replName;
}
public String getParagraphId() {
return paragraphId;
}

16
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java
View file

@ -17,6 +17,7 @@
package org.apache.zeppelin.interpreter;
import java.util.ArrayList;
import java.util.List;
/**
@ -96,6 +97,21 @@ public class InterpreterOption {
this.perNote = perNote;
}
public static InterpreterOption fromInterpreterOption(InterpreterOption other) {
InterpreterOption option = new InterpreterOption();
option.remote = other.remote;
option.host = other.host;
option.port = other.port;
option.perNote = other.perNote;
option.perUser = other.perUser;
option.isExistingProcess = other.isExistingProcess;
option.setPermission = other.setPermission;
option.users = (null == other.users) ?
new ArrayList<String>() : new ArrayList<>(other.users);
return option;
}
public boolean isRemote() {
return remote;
}

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -495,6 +495,7 @@ public class RemoteInterpreter extends Interpreter {
return new RemoteInterpreterContext(
ic.getNoteId(),
ic.getParagraphId(),
ic.getReplName(),
ic.getParagraphTitle(),
ic.getParagraphText(),
gson.toJson(ic.getAuthenticationInfo()),

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -532,6 +532,7 @@ public class RemoteInterpreterServer
return new InterpreterContext(
ric.getNoteId(),
ric.getParagraphId(),
ric.getReplName(),
ric.getParagraphTitle(),
ric.getParagraphText(),
gson.fromJson(ric.getAuthenticationInfo(), AuthenticationInfo.class),

17
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/InterpreterCompletion.java
View file

@ -1,20 +1,3 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
*

17
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java
View file

@ -1,20 +1,3 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
*

196
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java
View file

@ -1,20 +1,3 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
*
@ -57,12 +40,13 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)1);
private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)2);
private static final org.apache.thrift.protocol.TField PARAGRAPH_TITLE_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphTitle", org.apache.thrift.protocol.TType.STRING, (short)3);
private static final org.apache.thrift.protocol.TField PARAGRAPH_TEXT_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphText", org.apache.thrift.protocol.TType.STRING, (short)4);
private static final org.apache.thrift.protocol.TField AUTHENTICATION_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("authenticationInfo", org.apache.thrift.protocol.TType.STRING, (short)5);
private static final org.apache.thrift.protocol.TField CONFIG_FIELD_DESC = new org.apache.thrift.protocol.TField("config", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.protocol.TField GUI_FIELD_DESC = new org.apache.thrift.protocol.TField("gui", org.apache.thrift.protocol.TType.STRING, (short)7);
private static final org.apache.thrift.protocol.TField RUNNERS_FIELD_DESC = new org.apache.thrift.protocol.TField("runners", org.apache.thrift.protocol.TType.STRING, (short)8);
private static final org.apache.thrift.protocol.TField REPL_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("replName", org.apache.thrift.protocol.TType.STRING, (short)3);
private static final org.apache.thrift.protocol.TField PARAGRAPH_TITLE_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphTitle", org.apache.thrift.protocol.TType.STRING, (short)4);
private static final org.apache.thrift.protocol.TField PARAGRAPH_TEXT_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphText", org.apache.thrift.protocol.TType.STRING, (short)5);
private static final org.apache.thrift.protocol.TField AUTHENTICATION_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("authenticationInfo", org.apache.thrift.protocol.TType.STRING, (short)6);
private static final org.apache.thrift.protocol.TField CONFIG_FIELD_DESC = new org.apache.thrift.protocol.TField("config", org.apache.thrift.protocol.TType.STRING, (short)7);
private static final org.apache.thrift.protocol.TField GUI_FIELD_DESC = new org.apache.thrift.protocol.TField("gui", org.apache.thrift.protocol.TType.STRING, (short)8);
private static final org.apache.thrift.protocol.TField RUNNERS_FIELD_DESC = new org.apache.thrift.protocol.TField("runners", org.apache.thrift.protocol.TType.STRING, (short)9);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
static {
@ -72,6 +56,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
public String noteId; // required
public String paragraphId; // required
public String replName; // required
public String paragraphTitle; // required
public String paragraphText; // required
public String authenticationInfo; // required
@ -83,12 +68,13 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
NOTE_ID((short)1, "noteId"),
PARAGRAPH_ID((short)2, "paragraphId"),
PARAGRAPH_TITLE((short)3, "paragraphTitle"),
PARAGRAPH_TEXT((short)4, "paragraphText"),
AUTHENTICATION_INFO((short)5, "authenticationInfo"),
CONFIG((short)6, "config"),
GUI((short)7, "gui"),
RUNNERS((short)8, "runners");
REPL_NAME((short)3, "replName"),
PARAGRAPH_TITLE((short)4, "paragraphTitle"),
PARAGRAPH_TEXT((short)5, "paragraphText"),
AUTHENTICATION_INFO((short)6, "authenticationInfo"),
CONFIG((short)7, "config"),
GUI((short)8, "gui"),
RUNNERS((short)9, "runners");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@ -107,17 +93,19 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
return NOTE_ID;
case 2: // PARAGRAPH_ID
return PARAGRAPH_ID;
case 3: // PARAGRAPH_TITLE
case 3: // REPL_NAME
return REPL_NAME;
case 4: // PARAGRAPH_TITLE
return PARAGRAPH_TITLE;
case 4: // PARAGRAPH_TEXT
case 5: // PARAGRAPH_TEXT
return PARAGRAPH_TEXT;
case 5: // AUTHENTICATION_INFO
case 6: // AUTHENTICATION_INFO
return AUTHENTICATION_INFO;
case 6: // CONFIG
case 7: // CONFIG
return CONFIG;
case 7: // GUI
case 8: // GUI
return GUI;
case 8: // RUNNERS
case 9: // RUNNERS
return RUNNERS;
default:
return null;
@ -166,6 +154,8 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.PARAGRAPH_ID, new org.apache.thrift.meta_data.FieldMetaData("paragraphId", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.REPL_NAME, new org.apache.thrift.meta_data.FieldMetaData("replName", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.PARAGRAPH_TITLE, new org.apache.thrift.meta_data.FieldMetaData("paragraphTitle", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
tmpMap.put(_Fields.PARAGRAPH_TEXT, new org.apache.thrift.meta_data.FieldMetaData("paragraphText", org.apache.thrift.TFieldRequirementType.DEFAULT,
@ -188,6 +178,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
public RemoteInterpreterContext(
String noteId,
String paragraphId,
String replName,
String paragraphTitle,
String paragraphText,
String authenticationInfo,
@ -198,6 +189,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
this();
this.noteId = noteId;
this.paragraphId = paragraphId;
this.replName = replName;
this.paragraphTitle = paragraphTitle;
this.paragraphText = paragraphText;
this.authenticationInfo = authenticationInfo;
@ -216,6 +208,9 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
if (other.isSetParagraphId()) {
this.paragraphId = other.paragraphId;
}
if (other.isSetReplName()) {
this.replName = other.replName;
}
if (other.isSetParagraphTitle()) {
this.paragraphTitle = other.paragraphTitle;
}
@ -244,6 +239,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
public void clear() {
this.noteId = null;
this.paragraphId = null;
this.replName = null;
this.paragraphTitle = null;
this.paragraphText = null;
this.authenticationInfo = null;
@ -300,6 +296,30 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
}
}
public String getReplName() {
return this.replName;
}
public RemoteInterpreterContext setReplName(String replName) {
this.replName = replName;
return this;
}
public void unsetReplName() {
this.replName = null;
}
/** Returns true if field replName is set (has been assigned a value) and false otherwise */
public boolean isSetReplName() {
return this.replName != null;
}
public void setReplNameIsSet(boolean value) {
if (!value) {
this.replName = null;
}
}
public String getParagraphTitle() {
return this.paragraphTitle;
}
@ -462,6 +482,14 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
}
break;
case REPL_NAME:
if (value == null) {
unsetReplName();
} else {
setReplName((String)value);
}
break;
case PARAGRAPH_TITLE:
if (value == null) {
unsetParagraphTitle();
@ -521,6 +549,9 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
case PARAGRAPH_ID:
return getParagraphId();
case REPL_NAME:
return getReplName();
case PARAGRAPH_TITLE:
return getParagraphTitle();
@ -554,6 +585,8 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
return isSetNoteId();
case PARAGRAPH_ID:
return isSetParagraphId();
case REPL_NAME:
return isSetReplName();
case PARAGRAPH_TITLE:
return isSetParagraphTitle();
case PARAGRAPH_TEXT:
@ -601,6 +634,15 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
return false;
}
boolean this_present_replName = true && this.isSetReplName();
boolean that_present_replName = true && that.isSetReplName();
if (this_present_replName || that_present_replName) {
if (!(this_present_replName && that_present_replName))
return false;
if (!this.replName.equals(that.replName))
return false;
}
boolean this_present_paragraphTitle = true && this.isSetParagraphTitle();
boolean that_present_paragraphTitle = true && that.isSetParagraphTitle();
if (this_present_paragraphTitle || that_present_paragraphTitle) {
@ -672,6 +714,11 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
if (present_paragraphId)
list.add(paragraphId);
boolean present_replName = true && (isSetReplName());
list.add(present_replName);
if (present_replName)
list.add(replName);
boolean present_paragraphTitle = true && (isSetParagraphTitle());
list.add(present_paragraphTitle);
if (present_paragraphTitle)
@ -733,6 +780,16 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetReplName()).compareTo(other.isSetReplName());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetReplName()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.replName, other.replName);
if (lastComparison != 0) {
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetParagraphTitle()).compareTo(other.isSetParagraphTitle());
if (lastComparison != 0) {
return lastComparison;
@ -829,6 +886,14 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
}
first = false;
if (!first) sb.append(", ");
sb.append("replName:");
if (this.replName == null) {
sb.append("null");
} else {
sb.append(this.replName);
}
first = false;
if (!first) sb.append(", ");
sb.append("paragraphTitle:");
if (this.paragraphTitle == null) {
sb.append("null");
@ -935,7 +1000,15 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 3: // PARAGRAPH_TITLE
case 3: // REPL_NAME
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.replName = iprot.readString();
struct.setReplNameIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 4: // PARAGRAPH_TITLE
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.paragraphTitle = iprot.readString();
struct.setParagraphTitleIsSet(true);
@ -943,7 +1016,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 4: // PARAGRAPH_TEXT
case 5: // PARAGRAPH_TEXT
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.paragraphText = iprot.readString();
struct.setParagraphTextIsSet(true);
@ -951,7 +1024,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 5: // AUTHENTICATION_INFO
case 6: // AUTHENTICATION_INFO
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.authenticationInfo = iprot.readString();
struct.setAuthenticationInfoIsSet(true);
@ -959,7 +1032,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 6: // CONFIG
case 7: // CONFIG
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.config = iprot.readString();
struct.setConfigIsSet(true);
@ -967,7 +1040,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 7: // GUI
case 8: // GUI
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.gui = iprot.readString();
struct.setGuiIsSet(true);
@ -975,7 +1048,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
}
break;
case 8: // RUNNERS
case 9: // RUNNERS
if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
struct.runners = iprot.readString();
struct.setRunnersIsSet(true);
@ -1008,6 +1081,11 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
oprot.writeString(struct.paragraphId);
oprot.writeFieldEnd();
}
if (struct.replName != null) {
oprot.writeFieldBegin(REPL_NAME_FIELD_DESC);
oprot.writeString(struct.replName);
oprot.writeFieldEnd();
}
if (struct.paragraphTitle != null) {
oprot.writeFieldBegin(PARAGRAPH_TITLE_FIELD_DESC);
oprot.writeString(struct.paragraphTitle);
@ -1062,31 +1140,37 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
if (struct.isSetParagraphId()) {
optionals.set(1);
}
if (struct.isSetParagraphTitle()) {
if (struct.isSetReplName()) {
optionals.set(2);
}
if (struct.isSetParagraphText()) {
if (struct.isSetParagraphTitle()) {
optionals.set(3);
}
if (struct.isSetAuthenticationInfo()) {
if (struct.isSetParagraphText()) {
optionals.set(4);
}
if (struct.isSetConfig()) {
if (struct.isSetAuthenticationInfo()) {
optionals.set(5);
}
if (struct.isSetGui()) {
if (struct.isSetConfig()) {
optionals.set(6);
}
if (struct.isSetRunners()) {
if (struct.isSetGui()) {
optionals.set(7);
}
oprot.writeBitSet(optionals, 8);
if (struct.isSetRunners()) {
optionals.set(8);
}
oprot.writeBitSet(optionals, 9);
if (struct.isSetNoteId()) {
oprot.writeString(struct.noteId);
}
if (struct.isSetParagraphId()) {
oprot.writeString(struct.paragraphId);
}
if (struct.isSetReplName()) {
oprot.writeString(struct.replName);
}
if (struct.isSetParagraphTitle()) {
oprot.writeString(struct.paragraphTitle);
}
@ -1110,7 +1194,7 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
@Override
public void read(org.apache.thrift.protocol.TProtocol prot, RemoteInterpreterContext struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
BitSet incoming = iprot.readBitSet(8);
BitSet incoming = iprot.readBitSet(9);
if (incoming.get(0)) {
struct.noteId = iprot.readString();
struct.setNoteIdIsSet(true);
@ -1120,26 +1204,30 @@ public class RemoteInterpreterContext implements org.apache.thrift.TBase<RemoteI
struct.setParagraphIdIsSet(true);
}
if (incoming.get(2)) {
struct.replName = iprot.readString();
struct.setReplNameIsSet(true);
}
if (incoming.get(3)) {
struct.paragraphTitle = iprot.readString();
struct.setParagraphTitleIsSet(true);
}
if (incoming.get(3)) {
if (incoming.get(4)) {
struct.paragraphText = iprot.readString();
struct.setParagraphTextIsSet(true);
}
if (incoming.get(4)) {
if (incoming.get(5)) {
struct.authenticationInfo = iprot.readString();
struct.setAuthenticationInfoIsSet(true);
}
if (incoming.get(5)) {
if (incoming.get(6)) {
struct.config = iprot.readString();
struct.setConfigIsSet(true);
}
if (incoming.get(6)) {
if (incoming.get(7)) {
struct.gui = iprot.readString();
struct.setGuiIsSet(true);
}
if (incoming.get(7)) {
if (incoming.get(8)) {
struct.runners = iprot.readString();
struct.setRunnersIsSet(true);
}

17
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java
View file

@ -1,20 +1,3 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
*

17
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java
View file

@ -1,20 +1,3 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
*

17
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java
View file

@ -1,20 +1,3 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
*

17
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java
View file

@ -1,20 +1,3 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Autogenerated by Thrift Compiler (0.9.2)
*

14
zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift
View file

@ -18,16 +18,16 @@
namespace java org.apache.zeppelin.interpreter.thrift
struct RemoteInterpreterContext {
1: string noteId,
2: string paragraphId,
3: string paragraphTitle,
4: string paragraphText,
5: string authenticationInfo,
6: string config, // json serialized config
7: string gui, // json serialized gui
8: string runners // json serialized runner
3: string replName,
4: string paragraphTitle,
5: string paragraphText,
6: string authenticationInfo,
7: string config, // json serialized config
8: string gui, // json serialized gui
9: string runners // json serialized runner
}
struct RemoteInterpreterResultMessage {

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterContextTest.java
View file

@ -27,7 +27,7 @@ public class InterpreterContextTest {
public void testThreadLocal() {
assertNull(InterpreterContext.get());
InterpreterContext.set(new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null));
InterpreterContext.set(new InterpreterContext(null, null, null, null, null, null, null, null, null, null, null, null));
assertNotNull(InterpreterContext.get());
InterpreterContext.remove();

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/LazyOpenInterpreterTest.java
View file

@ -36,7 +36,7 @@ public class LazyOpenInterpreterTest {
assertFalse("Interpreter is not open", lazyOpenInterpreter.isOpen());
InterpreterContext interpreterContext =
new InterpreterContext("note", "id", "title", "text", null, null, null, null, null, null, null);
new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null);
lazyOpenInterpreter.interpret("intp 1", interpreterContext);
assertTrue("Interpeter is open", lazyOpenInterpreter.isOpen());
}

1
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
View file

@ -86,6 +86,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
context = new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),

1
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
View file

@ -85,6 +85,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
return new InterpreterContext(
"noteId",
"id",
null,
"title",
"text",
new AuthenticationInfo(),

10
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
View file

@ -149,6 +149,7 @@ public class RemoteInterpreterTest {
new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
@ -185,6 +186,7 @@ public class RemoteInterpreterTest {
new InterpreterContext(
"noteId",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
@ -245,6 +247,7 @@ public class RemoteInterpreterTest {
new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
@ -259,6 +262,7 @@ public class RemoteInterpreterTest {
new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
@ -313,6 +317,7 @@ public class RemoteInterpreterTest {
new InterpreterContext(
"note",
"jobA",
null,
"title",
"text",
new AuthenticationInfo(),
@ -349,6 +354,7 @@ public class RemoteInterpreterTest {
new InterpreterContext(
"note",
"jobB",
null,
"title",
"text",
new AuthenticationInfo(),
@ -415,6 +421,7 @@ public class RemoteInterpreterTest {
InterpreterResult ret = intpA.interpret(getJobName(), new InterpreterContext(
"note",
jobId,
null,
"title",
"text",
new AuthenticationInfo(),
@ -495,6 +502,7 @@ public class RemoteInterpreterTest {
InterpreterResult ret = intpA.interpret(stmt, new InterpreterContext(
"note",
jobId,
null,
"title",
"text",
new AuthenticationInfo(),
@ -597,6 +605,7 @@ public class RemoteInterpreterTest {
new InterpreterContext(
"note",
"jobA",
null,
"title",
"text",
new AuthenticationInfo(),
@ -756,6 +765,7 @@ public class RemoteInterpreterTest {
InterpreterContext context = new InterpreterContext(
"noteId",
"id",
null,
"title",
"text",
new AuthenticationInfo(),

1
zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
View file

@ -103,6 +103,7 @@ public class DistributedResourcePoolTest {
context = new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),

3
zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
View file

@ -109,6 +109,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
intpA.interpret("1000", new InterpreterContext(
"note",
"jobId",
null,
"title",
"text",
new AuthenticationInfo(),
@ -187,6 +188,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
InterpreterContext context = new InterpreterContext(
"note",
"jobId1",
null,
"title",
"text",
new AuthenticationInfo(),
@ -225,6 +227,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
InterpreterContext context = new InterpreterContext(
"note",
"jobId2",
null,
"title",
"text",
new AuthenticationInfo(),

1
zeppelin-server/pom.xml
View file

@ -504,5 +504,4 @@
</profile>
</profiles>
</project>

2
zeppelin-server/src/main/java/org/apache/zeppelin/server/ActiveDirectoryGroupRealm.java → zeppelin-server/src/main/java/org/apache/zeppelin/realm/ActiveDirectoryGroupRealm.java
View file

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.server;
package org.apache.zeppelin.realm;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;

2
zeppelin-server/src/main/java/org/apache/zeppelin/server/LdapGroupRealm.java → zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapGroupRealm.java
View file

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.server;
package org.apache.zeppelin.realm;
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.authz.SimpleAuthorizationInfo;

842
zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapRealm.java Normal file
View file

@ -0,0 +1,842 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.zeppelin.realm;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.authc.AuthenticationInfo;
import org.apache.shiro.authc.AuthenticationToken;
import org.apache.shiro.authc.SimpleAuthenticationInfo;
import org.apache.shiro.authc.credential.HashedCredentialsMatcher;
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.authz.SimpleAuthorizationInfo;
import org.apache.shiro.crypto.hash.DefaultHashService;
import org.apache.shiro.crypto.hash.Hash;
import org.apache.shiro.crypto.hash.HashRequest;
import org.apache.shiro.crypto.hash.HashService;
import org.apache.shiro.realm.ldap.JndiLdapRealm;
import org.apache.shiro.realm.ldap.LdapContextFactory;
import org.apache.shiro.realm.ldap.LdapUtils;
import org.apache.shiro.subject.MutablePrincipalCollection;
import org.apache.shiro.subject.PrincipalCollection;
import org.apache.shiro.util.StringUtils;
import org.mortbay.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.naming.AuthenticationException;
import javax.naming.Context;
import javax.naming.NamingEnumeration;
import javax.naming.NamingException;
import javax.naming.PartialResultException;
import javax.naming.SizeLimitExceededException;
import javax.naming.directory.Attribute;
import javax.naming.directory.SearchControls;
import javax.naming.directory.SearchResult;
import javax.naming.ldap.Control;
import javax.naming.ldap.LdapContext;
import javax.naming.ldap.LdapName;
import javax.naming.ldap.PagedResultsControl;
/**
* Implementation of {@link org.apache.shiro.realm.ldap.JndiLdapRealm} that also
* returns each user's groups. This implementation is heavily based on
* org.apache.isis.security.shiro.IsisLdapRealm.
*
* <p>This implementation saves looked up ldap groups in Shiro Session to make them
* easy to be looked up outside of this object
*
* <p>Sample config for <tt>shiro.ini</tt>:
*
* <p>[main]
* ldapRealm=org.apache.zeppelin.realm.LdapRealm
* ldapRealm.contextFactory=$ldapGroupContextFactory
* ldapRealm.contextFactory.authenticationMechanism=simple
* ldapRealm.contextFactory.url=ldap://localhost:33389
* ldapRealm.userDnTemplate=uid={0},ou=people,dc=hadoop,dc=apache,dc=org
* # Ability to set ldap paging Size if needed default is 100
* ldapRealm.pagingSize = 200
* ldapRealm.authorizationEnabled=true
* ldapRealm.contextFactory.systemAuthenticationMechanism=simple
* ldapRealm.searchBase=dc=hadoop,dc=apache,dc=org
* ldapRealm.userSearchBase = dc=hadoop,dc=apache,dc=org
* ldapRealm.groupSearchBase = ou=groups,dc=hadoop,dc=apache,dc=org
* ldapRealm.groupObjectClass=groupofnames
* # Allow userSearchAttribute to be customized
* ldapRealm.userSearchAttributeName = sAMAccountName
* ldapRealm.memberAttribute=member
* # force usernames returned from ldap to lowercase useful for AD
* ldapRealm.userLowerCase = true
* # ability set searchScopes subtree (default), one, base
* ldapRealm.userSearchScope = subtree;
* ldapRealm.groupSearchScope = subtree;
* ldapRealm.memberAttributeValueTemplate=cn={0},ou=people,dc=hadoop,dc=apache,
* dc=org
* ldapRealm.contextFactory.systemUsername=uid=guest,ou=people,dc=hadoop,dc=
* apache,dc=org
* ldapRealm.contextFactory.systemPassword=S{ALIAS=ldcSystemPassword} [urls]
* **=authcBasic
*
* <p># optional mapping from physical groups to logical application roles
* ldapRealm.rolesByGroup = \ LDN_USERS: user_role,\ NYK_USERS: user_role,\
* HKG_USERS: user_role,\ GLOBAL_ADMIN: admin_role,\ DEMOS: self-install_role
*
* <p>ldapRealm.permissionsByRole=\ user_role = *:ToDoItemsJdo:*:*,\
* *:ToDoItem:*:*; \ self-install_role = *:ToDoItemsFixturesService:install:* ;
* \ admin_role = *
*
* <p>securityManager.realms = $ldapRealm
*
*/
public class LdapRealm extends JndiLdapRealm {
private static final SearchControls SUBTREE_SCOPE = new SearchControls();
private static final SearchControls ONELEVEL_SCOPE = new SearchControls();
private static final SearchControls OBJECT_SCOPE = new SearchControls();
private static final String SUBJECT_USER_ROLES = "subject.userRoles";
private static final String SUBJECT_USER_GROUPS = "subject.userGroups";
private static final String MEMBER_URL = "memberUrl";
private static final String POSIX_GROUP = "posixGroup";
private static Pattern TEMPLATE_PATTERN = Pattern.compile("\\{(\\d+?)\\}");
private static String DEFAULT_PRINCIPAL_REGEX = "(.*)";
private static final String MEMBER_SUBSTITUTION_TOKEN = "{0}";
private static final String HASHING_ALGORITHM = "SHA-1";
private static final Logger log = LoggerFactory.getLogger(LdapRealm.class);
static {
SUBTREE_SCOPE.setSearchScope(SearchControls.SUBTREE_SCOPE);
ONELEVEL_SCOPE.setSearchScope(SearchControls.ONELEVEL_SCOPE);
OBJECT_SCOPE.setSearchScope(SearchControls.OBJECT_SCOPE);
}
private String searchBase;
private String userSearchBase;
private int pagingSize = 100;
private boolean userLowerCase;
private String principalRegex = DEFAULT_PRINCIPAL_REGEX;
private Pattern principalPattern = Pattern.compile(DEFAULT_PRINCIPAL_REGEX);
private String userDnTemplate = "{0}";
private String userSearchFilter = null;
private String userSearchAttributeTemplate = "{0}";
private String userSearchScope = "subtree";
private String groupSearchScope = "subtree";
private String groupSearchBase;
private String groupObjectClass = "groupOfNames";
// typical value: member, uniqueMember, meberUrl
private String memberAttribute = "member";
private String groupIdAttribute = "cn";
private String memberAttributeValuePrefix = "uid={0}";
private String memberAttributeValueSuffix = "";
private final Map<String, String> rolesByGroup = new LinkedHashMap<String, String>();
private final Map<String, List<String>> permissionsByRole =
new LinkedHashMap<String, List<String>>();
private boolean authorizationEnabled;
private String userSearchAttributeName;
private String userObjectClass = "person";
private HashService hashService = new DefaultHashService();
public LdapRealm() {
HashedCredentialsMatcher credentialsMatcher = new HashedCredentialsMatcher(HASHING_ALGORITHM);
setCredentialsMatcher(credentialsMatcher);
}
@Override
protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token)
throws org.apache.shiro.authc.AuthenticationException {
try {
return super.doGetAuthenticationInfo(token);
} catch (org.apache.shiro.authc.AuthenticationException ae) {
throw ae;
}
}
/**
* Get groups from LDAP.
*
* @param principals
* the principals of the Subject whose AuthenticationInfo should
* be queried from the LDAP server.
* @param ldapContextFactory
* factory used to retrieve LDAP connections.
* @return an {@link AuthorizationInfo} instance containing information
* retrieved from the LDAP server.
* @throws NamingException
* if any LDAP errors occur during the search.
*/
@Override
protected AuthorizationInfo queryForAuthorizationInfo(final PrincipalCollection principals,
final LdapContextFactory ldapContextFactory) throws NamingException {
if (!isAuthorizationEnabled()) {
return null;
}
final Set<String> roleNames = getRoles(principals, ldapContextFactory);
if (log.isDebugEnabled()) {
log.debug("RolesNames Authorization: " + roleNames);
}
SimpleAuthorizationInfo simpleAuthorizationInfo = new SimpleAuthorizationInfo(roleNames);
Set<String> stringPermissions = permsFor(roleNames);
simpleAuthorizationInfo.setStringPermissions(stringPermissions);
return simpleAuthorizationInfo;
}
private Set<String> getRoles(PrincipalCollection principals,
final LdapContextFactory ldapContextFactory)
throws NamingException {
final String username = (String) getAvailablePrincipal(principals);
LdapContext systemLdapCtx = null;
try {
systemLdapCtx = ldapContextFactory.getSystemLdapContext();
return rolesFor(principals, username, systemLdapCtx, ldapContextFactory);
} catch (AuthenticationException ae) {
ae.printStackTrace();
return Collections.emptySet();
} finally {
LdapUtils.closeContext(systemLdapCtx);
}
}
private Set<String> rolesFor(PrincipalCollection principals,
String userNameIn, final LdapContext ldapCtx,
final LdapContextFactory ldapContextFactory) throws NamingException {
final Set<String> roleNames = new HashSet<>();
final Set<String> groupNames = new HashSet<>();
final String userName;
if (getUserLowerCase()) {
log.debug("userLowerCase true");
userName = userNameIn.toLowerCase();
} else {
userName = userNameIn;
}
String userDn;
if (userSearchAttributeName == null || userSearchAttributeName.isEmpty()) {
// memberAttributeValuePrefix and memberAttributeValueSuffix
// were computed from memberAttributeValueTemplate
userDn = memberAttributeValuePrefix + userName + memberAttributeValueSuffix;
} else {
userDn = getUserDn(userName);
}
// Activate paged results
int pageSize = getPagingSize();
if (log.isDebugEnabled()) {
log.debug("Ldap PagingSize: " + pageSize);
}
int numResults = 0;
byte[] cookie = null;
try {
ldapCtx.addToEnvironment(Context.REFERRAL, "ignore");
ldapCtx.setRequestControls(new Control[]{new PagedResultsControl(pageSize,
Control.NONCRITICAL)});
do {
// ldapsearch -h localhost -p 33389 -D
// uid=guest,ou=people,dc=hadoop,dc=apache,dc=org -w guest-password
// -b dc=hadoop,dc=apache,dc=org -s sub '(objectclass=*)'
NamingEnumeration<SearchResult> searchResultEnum = null;
SearchControls searchControls = getGroupSearchControls();
try {
searchResultEnum = ldapCtx.search(
getGroupSearchBase(),
"objectClass=" + groupObjectClass,
searchControls);
while (searchResultEnum != null && searchResultEnum.hasMore()) {
// searchResults contains all the groups in search scope
numResults++;
final SearchResult group = searchResultEnum.next();
addRoleIfMember(userDn, group, roleNames, groupNames, ldapContextFactory);
}
} catch (PartialResultException e) {
log.debug("Ignoring PartitalResultException");
} finally {
if (searchResultEnum != null) {
searchResultEnum.close();
}
}
// Re-activate paged results
ldapCtx.setRequestControls(new Control[]{new PagedResultsControl(pageSize,
cookie, Control.CRITICAL)});
} while (cookie != null);
} catch (SizeLimitExceededException e) {
log.info("Only retrieved first " + numResults +
" groups due to SizeLimitExceededException.");
} catch (IOException e) {
log.error("Unabled to setup paged results");
}
// save role names and group names in session so that they can be
// easily looked up outside of this object
SecurityUtils.getSubject().getSession().setAttribute(SUBJECT_USER_ROLES, roleNames);
SecurityUtils.getSubject().getSession().setAttribute(SUBJECT_USER_GROUPS, groupNames);
if (!groupNames.isEmpty() && (principals instanceof MutablePrincipalCollection)) {
((MutablePrincipalCollection) principals).addAll(groupNames, getName());
}
if (log.isDebugEnabled()) {
log.debug("User RoleNames: " + userName + "::" + roleNames);
}
return roleNames;
}
private void addRoleIfMember(final String userDn, final SearchResult group,
final Set<String> roleNames, final Set<String> groupNames,
final LdapContextFactory ldapContextFactory) throws NamingException {
NamingEnumeration<? extends Attribute> attributeEnum = null;
NamingEnumeration<?> ne = null;
try {
LdapName userLdapDn = new LdapName(userDn);
Attribute attribute = group.getAttributes().get(getGroupIdAttribute());
String groupName = attribute.get().toString();
attributeEnum = group.getAttributes().getAll();
while (attributeEnum.hasMore()) {
final Attribute attr = attributeEnum.next();
if (!memberAttribute.equalsIgnoreCase(attr.getID())) {
continue;
}
ne = attr.getAll();
while (ne.hasMore()) {
String attrValue = ne.next().toString();
if (memberAttribute.equalsIgnoreCase(MEMBER_URL)) {
boolean dynamicGroupMember = isUserMemberOfDynamicGroup(userLdapDn, attrValue,
ldapContextFactory);
if (dynamicGroupMember) {
groupNames.add(groupName);
String roleName = roleNameFor(groupName);
if (roleName != null) {
roleNames.add(roleName);
} else {
roleNames.add(groupName);
}
}
} else {
if (groupObjectClass.equalsIgnoreCase(POSIX_GROUP)) {
attrValue = memberAttributeValuePrefix + attrValue + memberAttributeValueSuffix;
}
if (userLdapDn.equals(new LdapName(attrValue))) {
groupNames.add(groupName);
String roleName = roleNameFor(groupName);
if (roleName != null) {
roleNames.add(roleName);
} else {
roleNames.add(groupName);
}
break;
}
}
}
}
} finally {
try {
if (attributeEnum != null) {
attributeEnum.close();
}
} finally {
if (ne != null) {
ne.close();
}
}
}
}
public Map<String, String> getListRoles() {
Map<String, String> groupToRoles = getRolesByGroup();
Map<String, String> roles = new HashMap<>();
for (Map.Entry<String, String> entry : groupToRoles.entrySet()){
roles.put(entry.getValue(), entry.getKey());
}
return roles;
}
private String roleNameFor(String groupName) {
return !rolesByGroup.isEmpty() ? rolesByGroup.get(groupName) : groupName;
}
private Set<String> permsFor(Set<String> roleNames) {
Set<String> perms = new LinkedHashSet<String>(); // preserve order
for (String role : roleNames) {
List<String> permsForRole = permissionsByRole.get(role);
if (log.isDebugEnabled()) {
log.debug("PermsForRole: " + role);
log.debug("PermByRole: " + permsForRole);
}
if (permsForRole != null) {
perms.addAll(permsForRole);
}
}
return perms;
}
public String getSearchBase() {
return searchBase;
}
public void setSearchBase(String searchBase) {
this.searchBase = searchBase;
}
public String getUserSearchBase() {
return (userSearchBase != null && !userSearchBase.isEmpty()) ? userSearchBase : searchBase;
}
public void setUserSearchBase(String userSearchBase) {
this.userSearchBase = userSearchBase;
}
public int getPagingSize() {
return pagingSize;
}
public void setPagingSize(int pagingSize) {
this.pagingSize = pagingSize;
}
public String getGroupSearchBase() {
return (groupSearchBase != null && !groupSearchBase.isEmpty()) ? groupSearchBase : searchBase;
}
public void setGroupSearchBase(String groupSearchBase) {
this.groupSearchBase = groupSearchBase;
}
public String getGroupObjectClass() {
return groupObjectClass;
}
public void setGroupObjectClass(String groupObjectClassAttribute) {
this.groupObjectClass = groupObjectClassAttribute;
}
public String getMemberAttribute() {
return memberAttribute;
}
public void setMemberAttribute(String memberAttribute) {
this.memberAttribute = memberAttribute;
}
public String getGroupIdAttribute() {
return groupIdAttribute;
}
public void setGroupIdAttribute(String groupIdAttribute) {
this.groupIdAttribute = groupIdAttribute;
}
/**
* Set Member Attribute Template for LDAP.
*
* @param template
* DN template to be used to query ldap.
* @throws IllegalArgumentException
* if template is empty or null.
*/
public void setMemberAttributeValueTemplate(String template) {
if (!StringUtils.hasText(template)) {
String msg = "User DN template cannot be null or empty.";
throw new IllegalArgumentException(msg);
}
int index = template.indexOf(MEMBER_SUBSTITUTION_TOKEN);
if (index < 0) {
String msg = "Member attribute value template must contain the '" + MEMBER_SUBSTITUTION_TOKEN
+ "' replacement token to understand how to " + "parse the group members.";
throw new IllegalArgumentException(msg);
}
String prefix = template.substring(0, index);
String suffix = template.substring(prefix.length() + MEMBER_SUBSTITUTION_TOKEN.length());
this.memberAttributeValuePrefix = prefix;
this.memberAttributeValueSuffix = suffix;
}
public void setRolesByGroup(Map<String, String> rolesByGroup) {
this.rolesByGroup.putAll(rolesByGroup);
}
public Map<String, String> getRolesByGroup() {
return rolesByGroup;
}
public void setPermissionsByRole(String permissionsByRoleStr) {
permissionsByRole.putAll(parsePermissionByRoleString(permissionsByRoleStr));
}
public Map<String, List<String>> getPermissionsByRole() {
return permissionsByRole;
}
public boolean isAuthorizationEnabled() {
return authorizationEnabled;
}
public void setAuthorizationEnabled(boolean authorizationEnabled) {
this.authorizationEnabled = authorizationEnabled;
}
public String getUserSearchAttributeName() {
return userSearchAttributeName;
}
/**
* Set User Search Attribute Name for LDAP.
*
* @param userSearchAttributeName
* userAttribute to search ldap.
*/
public void setUserSearchAttributeName(String userSearchAttributeName) {
if (userSearchAttributeName != null) {
userSearchAttributeName = userSearchAttributeName.trim();
}
this.userSearchAttributeName = userSearchAttributeName;
}
public String getUserObjectClass() {
return userObjectClass;
}
public void setUserObjectClass(String userObjectClass) {
this.userObjectClass = userObjectClass;
}
private Map<String, List<String>> parsePermissionByRoleString(String permissionsByRoleStr) {
Map<String, List<String>> perms = new HashMap<String, List<String>>();
// split by semicolon ; then by eq = then by comma ,
StringTokenizer stSem = new StringTokenizer(permissionsByRoleStr, ";");
while (stSem.hasMoreTokens()) {
String roleAndPerm = stSem.nextToken();
StringTokenizer stEq = new StringTokenizer(roleAndPerm, "=");
if (stEq.countTokens() != 2) {
continue;
}
String role = stEq.nextToken().trim();
String perm = stEq.nextToken().trim();
StringTokenizer stCom = new StringTokenizer(perm, ",");
List<String> permList = new ArrayList<String>();
while (stCom.hasMoreTokens()) {
permList.add(stCom.nextToken().trim());
}
perms.put(role, permList);
}
return perms;
}
boolean isUserMemberOfDynamicGroup(LdapName userLdapDn, String memberUrl,
final LdapContextFactory ldapContextFactory) throws NamingException {
// ldap://host:port/dn?attributes?scope?filter?extensions
if (memberUrl == null) {
return false;
}
String[] tokens = memberUrl.split("\\?");
if (tokens.length < 4) {
return false;
}
String searchBaseString = tokens[0].substring(tokens[0].lastIndexOf("/") + 1);
String searchScope = tokens[2];
String searchFilter = tokens[3];
LdapName searchBaseDn = new LdapName(searchBaseString);
// do scope test
if (searchScope.equalsIgnoreCase("base")) {
log.debug("DynamicGroup SearchScope base");
return false;
}
if (!userLdapDn.toString().endsWith(searchBaseDn.toString())) {
return false;
}
if (searchScope.equalsIgnoreCase("one") && (userLdapDn.size() != searchBaseDn.size() - 1)) {
log.debug("DynamicGroup SearchScope one");
return false;
}
// search for the filter, substituting base with userDn
// search for base_dn=userDn, scope=base, filter=filter
LdapContext systemLdapCtx = null;
systemLdapCtx = ldapContextFactory.getSystemLdapContext();
boolean member = false;
NamingEnumeration<SearchResult> searchResultEnum = null;
try {
searchResultEnum = systemLdapCtx.search(userLdapDn, searchFilter,
searchScope.equalsIgnoreCase("sub") ? SUBTREE_SCOPE : ONELEVEL_SCOPE);
if (searchResultEnum.hasMore()) {
return true;
}
} finally {
try {
if (searchResultEnum != null) {
searchResultEnum.close();
}
} finally {
LdapUtils.closeContext(systemLdapCtx);
}
}
return member;
}
public String getPrincipalRegex() {
return principalRegex;
}
/**
* Set Regex for Principal LDAP.
*
* @param regex
* regex to use to search for principal in shiro.
*/
public void setPrincipalRegex(String regex) {
if (regex == null || regex.trim().isEmpty()) {
principalPattern = Pattern.compile(DEFAULT_PRINCIPAL_REGEX);
principalRegex = DEFAULT_PRINCIPAL_REGEX;
} else {
regex = regex.trim();
Pattern pattern = Pattern.compile(regex);
principalPattern = pattern;
principalRegex = regex;
}
}
public String getUserSearchAttributeTemplate() {
return userSearchAttributeTemplate;
}
public void setUserSearchAttributeTemplate(final String template) {
this.userSearchAttributeTemplate = (template == null ? null : template.trim());
}
public String getUserSearchFilter() {
return userSearchFilter;
}
public void setUserSearchFilter(final String filter) {
this.userSearchFilter = (filter == null ? null : filter.trim());
}
public boolean getUserLowerCase() {
return userLowerCase;
}
public void setUserLowerCase(boolean userLowerCase) {
this.userLowerCase = userLowerCase;
}
public String getUserSearchScope() {
return userSearchScope;
}
public void setUserSearchScope(final String scope) {
this.userSearchScope = (scope == null ? null : scope.trim().toLowerCase());
}
public String getGroupSearchScope() {
return groupSearchScope;
}
public void setGroupSearchScope(final String scope) {
this.groupSearchScope = (scope == null ? null : scope.trim().toLowerCase());
}
private SearchControls getUserSearchControls() {
SearchControls searchControls = SUBTREE_SCOPE;
if ("onelevel".equalsIgnoreCase(userSearchScope)) {
searchControls = ONELEVEL_SCOPE;
} else if ("object".equalsIgnoreCase(userSearchScope)) {
searchControls = OBJECT_SCOPE;
}
return searchControls;
}
private SearchControls getGroupSearchControls() {
SearchControls searchControls = SUBTREE_SCOPE;
if ("onelevel".equalsIgnoreCase(groupSearchScope)) {
searchControls = ONELEVEL_SCOPE;
} else if ("object".equalsIgnoreCase(groupSearchScope)) {
searchControls = OBJECT_SCOPE;
}
return searchControls;
}
@Override
public void setUserDnTemplate(final String template) throws IllegalArgumentException {
userDnTemplate = template;
}
private Matcher matchPrincipal(final String principal) {
Matcher matchedPrincipal = principalPattern.matcher(principal);
if (!matchedPrincipal.matches()) {
throw new IllegalArgumentException("Principal "
+ principal + " does not match " + principalRegex);
}
return matchedPrincipal;
}
/**
* Returns the LDAP User Distinguished Name (DN) to use when acquiring an
* {@link javax.naming.ldap.LdapContext LdapContext} from the
* {@link LdapContextFactory}.
* <p/>
* If the the {@link #getUserDnTemplate() userDnTemplate} property has been
* set, this implementation will construct the User DN by substituting the
* specified {@code principal} into the configured template. If the
* {@link #getUserDnTemplate() userDnTemplate} has not been set, the method
* argument will be returned directly (indicating that the submitted
* authentication token principal <em>is</em> the User DN).
*
* @param principal
* the principal to substitute into the configured
* {@link #getUserDnTemplate() userDnTemplate}.
* @return the constructed User DN to use at runtime when acquiring an
* {@link javax.naming.ldap.LdapContext}.
* @throws IllegalArgumentException
* if the method argument is null or empty
* @throws IllegalStateException
* if the {@link #getUserDnTemplate userDnTemplate} has not been
* set.
* @see LdapContextFactory#getLdapContext(Object, Object)
*/
@Override
protected String getUserDn(final String principal) throws IllegalArgumentException,
IllegalStateException {
String userDn;
Matcher matchedPrincipal = matchPrincipal(principal);
String userSearchBase = getUserSearchBase();
String userSearchAttributeName = getUserSearchAttributeName();
// If not searching use the userDnTemplate and return.
if ((userSearchBase == null || userSearchBase.isEmpty()) || (userSearchAttributeName == null
&& userSearchFilter == null && !"object".equalsIgnoreCase(userSearchScope))) {
userDn = expandTemplate(userDnTemplate, matchedPrincipal);
if (log.isDebugEnabled()) {
log.debug("LDAP UserDN and Principal: " + userDn + "," + principal);
}
return userDn;
}
// Create the searchBase and searchFilter from config.
String searchBase = expandTemplate(getUserSearchBase(), matchedPrincipal);
String searchFilter = null;
if (userSearchFilter == null) {
if (userSearchAttributeName == null) {
searchFilter = String.format("(objectclass=%1$s)", getUserObjectClass());
} else {
searchFilter = String.format("(&(objectclass=%1$s)(%2$s=%3$s))", getUserObjectClass(),
userSearchAttributeName, expandTemplate(getUserSearchAttributeTemplate(),
matchedPrincipal));
}
} else {
searchFilter = expandTemplate(userSearchFilter, matchedPrincipal);
}
SearchControls searchControls = getUserSearchControls();
// Search for userDn and return.
LdapContext systemLdapCtx = null;
NamingEnumeration<SearchResult> searchResultEnum = null;
try {
systemLdapCtx = getContextFactory().getSystemLdapContext();
if (log.isDebugEnabled()) {
log.debug("SearchBase,SearchFilter,UserSearchScope: " + searchBase
+ "," + searchFilter + "," + userSearchScope);
}
searchResultEnum = systemLdapCtx.search(searchBase, searchFilter, searchControls);
// SearchResults contains all the entries in search scope
if (searchResultEnum.hasMore()) {
SearchResult searchResult = searchResultEnum.next();
userDn = searchResult.getNameInNamespace();
if (log.isDebugEnabled()) {
log.debug("UserDN Returned,Principal: " + userDn + "," + principal);
}
return userDn;
} else {
throw new IllegalArgumentException("Illegal principal name: " + principal);
}
} catch (AuthenticationException ne) {
ne.printStackTrace();
throw new IllegalArgumentException("Illegal principal name: " + principal);
} catch (NamingException ne) {
throw new IllegalArgumentException("Hit NamingException: " + ne.getMessage());
} finally {
try {
if (searchResultEnum != null) {
searchResultEnum.close();
}
} catch (NamingException ne) {
// Ignore exception on close.
} finally {
LdapUtils.closeContext(systemLdapCtx);
}
}
}
@Override
protected AuthenticationInfo createAuthenticationInfo(AuthenticationToken token,
Object ldapPrincipal,
Object ldapCredentials, LdapContext ldapContext) throws NamingException {
HashRequest.Builder builder = new HashRequest.Builder();
Hash credentialsHash = hashService
.computeHash(builder.setSource(token.getCredentials())
.setAlgorithmName(HASHING_ALGORITHM).build());
return new SimpleAuthenticationInfo(token.getPrincipal(),
credentialsHash.toHex(), credentialsHash.getSalt(),
getName());
}
private static final String expandTemplate(final String template, final Matcher input) {
String output = template;
Matcher matcher = TEMPLATE_PATTERN.matcher(output);
while (matcher.find()) {
String lookupStr = matcher.group(1);
int lookupIndex = Integer.parseInt(lookupStr);
String lookupValue = input.group(lookupIndex);
output = matcher.replaceFirst(lookupValue == null ? "" : lookupValue);
matcher = TEMPLATE_PATTERN.matcher(output);
}
return output;
}
}

11
zeppelin-server/src/main/java/org/apache/zeppelin/realm/ZeppelinHubRealm.java
View file

@ -37,6 +37,7 @@ import org.apache.shiro.authc.UsernamePasswordToken;
import org.apache.shiro.authz.AuthorizationInfo;
import org.apache.shiro.realm.AuthorizingRealm;
import org.apache.shiro.subject.PrincipalCollection;
import org.apache.zeppelin.notebook.repo.zeppelinhub.model.UserSessionContainer;
import org.apache.zeppelin.server.ZeppelinServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -56,6 +57,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
private static final String USER_LOGIN_API_ENDPOINT = "api/v1/users/login";
private static final String JSON_CONTENT_TYPE = "application/json";
private static final String UTF_8_ENCODING = "UTF-8";
private static final String USER_SESSION_HEADER = "X-session";
private static final AtomicInteger INSTANCE_COUNT = new AtomicInteger();
private final HttpClient httpClient;
@ -126,6 +128,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
protected User authenticateUser(String requestBody) {
PutMethod put = new PutMethod(Joiner.on("/").join(zeppelinhubUrl, USER_LOGIN_API_ENDPOINT));
String responseBody = StringUtils.EMPTY;
String userSession = StringUtils.EMPTY;
try {
put.setRequestEntity(new StringRequestEntity(requestBody, JSON_CONTENT_TYPE, UTF_8_ENCODING));
int statusCode = httpClient.executeMethod(put);
@ -136,6 +139,7 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
+ "Login or password incorrect");
}
responseBody = put.getResponseBodyAsString();
userSession = put.getResponseHeader(USER_SESSION_HEADER).getValue();
put.releaseConnection();
} catch (IOException e) {
@ -150,13 +154,16 @@ public class ZeppelinHubRealm extends AuthorizingRealm {
LOG.error("Cannot deserialize ZeppelinHub response to User instance", e);
throw new AuthenticationException("Cannot login to ZeppelinHub");
}
// Add ZeppelinHub user_session token this singleton map, this will help ZeppelinHubRepo
// to get specific information about the current user.
UserSessionContainer.instance.setSession(account.login, userSession);
/* TODO(khalid): add proper roles and add listener */
HashSet<String> userAndRoles = new HashSet<String>();
userAndRoles.add(account.login);
ZeppelinServer.notebookWsServer.broadcastReloadedNoteList(
new org.apache.zeppelin.user.AuthenticationInfo(account.login), userAndRoles);
return account;
}

71
zeppelin-server/src/main/java/org/apache/zeppelin/rest/GetUserList.java
View file

@ -24,7 +24,8 @@ import org.apache.shiro.realm.ldap.JndiLdapContextFactory;
import org.apache.shiro.realm.ldap.JndiLdapRealm;
import org.apache.shiro.realm.text.IniRealm;
import org.apache.shiro.util.JdbcUtils;
import org.apache.zeppelin.server.ActiveDirectoryGroupRealm;
import org.apache.zeppelin.realm.ActiveDirectoryGroupRealm;
import org.apache.zeppelin.realm.LdapRealm;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -114,8 +115,76 @@ public class GetUserList {
} catch (Exception e) {
LOG.error("Error retrieving User list from Ldap Realm", e);
}
LOG.info("UserList: " + userList);
return userList;
}
/**
* function to extract users from Zeppelin LdapRealm
*/
public List<String> getUserList(LdapRealm r, String searchText) {
List<String> userList = new ArrayList<>();
if (LOG.isDebugEnabled()) {
LOG.debug("SearchText: " + searchText);
}
String userAttribute = r.getUserSearchAttributeName();
String userSearchRealm = r.getUserSearchBase();
String userObjectClass = r.getUserObjectClass();
JndiLdapContextFactory CF = (JndiLdapContextFactory) r.getContextFactory();
try {
LdapContext ctx = CF.getSystemLdapContext();
SearchControls constraints = new SearchControls();
constraints.setSearchScope(SearchControls.SUBTREE_SCOPE);
String[] attrIDs = {userAttribute};
constraints.setReturningAttributes(attrIDs);
NamingEnumeration result = ctx.search(userSearchRealm, "(&(objectclass=" +
userObjectClass + ")("
+ userAttribute + "=" + searchText + "))", constraints);
while (result.hasMore()) {
Attributes attrs = ((SearchResult) result.next()).getAttributes();
if (attrs.get(userAttribute) != null) {
String currentUser;
if (r.getUserLowerCase()) {
LOG.debug("userLowerCase true");
currentUser = ((String) attrs.get(userAttribute).get()).toLowerCase();
} else {
LOG.debug("userLowerCase false");
currentUser = (String) attrs.get(userAttribute).get();
}
if (LOG.isDebugEnabled()) {
LOG.debug("CurrentUser: " + currentUser);
}
userList.add(currentUser.trim());
}
}
} catch (Exception e) {
LOG.error("Error retrieving User list from Ldap Realm", e);
}
return userList;
}
/***
* Get user roles from shiro.ini for Zeppelin LdapRealm
* @param r
* @return
*/
public List<String> getRolesList(LdapRealm r) {
List<String> roleList = new ArrayList<>();
Map<String, String> roles = r.getListRoles();
if (roles != null) {
Iterator it = roles.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
if (LOG.isDebugEnabled()) {
LOG.debug("RoleKeyValue: " + pair.getKey() +
" = " + pair.getValue());
}
roleList.add((String) pair.getKey());
}
}
return roleList;
}
public List<String> getUserList(ActiveDirectoryGroupRealm r, String searchText) {
List<String> userList = new ArrayList<>();

25
zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
View file

@ -80,6 +80,27 @@ public class InterpreterRestApi {
return new JsonResponse<>(Status.OK, "", interpreterFactory.get()).build();
}
/**
* Get a setting
*/
@GET
@Path("setting/{settingId}")
@ZeppelinApi
public Response getSetting(@PathParam("settingId") String settingId) {
try {
InterpreterSetting setting = interpreterFactory.get(settingId);
if (setting == null) {
return new JsonResponse<>(Status.NOT_FOUND).build();
} else {
return new JsonResponse<>(Status.OK, "", setting).build();
}
} catch (NullPointerException e) {
logger.error("Exception in InterpreterRestApi while creating ", e);
return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(),
ExceptionUtils.getStackTrace(e)).build();
}
}
/**
* Add new interpreter setting
*
@ -209,7 +230,7 @@ public class InterpreterRestApi {
try {
Repository request = gson.fromJson(message, Repository.class);
interpreterFactory.addRepository(request.getId(), request.getUrl(), request.isSnapshot(),
request.getAuthentication(), request.getProxy());
request.getAuthentication(), request.getProxy());
logger.info("New repository {} added", request.getId());
} catch (Exception e) {
logger.error("Exception in InterpreterRestApi while adding repository ", e);
@ -225,7 +246,7 @@ public class InterpreterRestApi {
@GET
@Path("getmetainfos/{settingId}")
public Response getMetaInfo(@Context HttpServletRequest req,
@PathParam("settingId") String settingId) {
@PathParam("settingId") String settingId) {
String propName = req.getParameter("propName");
if (propName == null) {
return new JsonResponse<>(Status.BAD_REQUEST).build();

19
zeppelin-server/src/main/java/org/apache/zeppelin/rest/SecurityRestApi.java
View file

@ -25,7 +25,8 @@ import org.apache.shiro.realm.ldap.JndiLdapRealm;
import org.apache.shiro.realm.text.IniRealm;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.server.ActiveDirectoryGroupRealm;
import org.apache.zeppelin.realm.ActiveDirectoryGroupRealm;
import org.apache.zeppelin.realm.LdapRealm;
import org.apache.zeppelin.server.JsonResponse;
import org.apache.zeppelin.ticket.TicketContainer;
import org.apache.zeppelin.utils.SecurityUtils;
@ -105,16 +106,22 @@ public class SecurityRestApi {
if (realmsList != null) {
for (Iterator<Realm> iterator = realmsList.iterator(); iterator.hasNext(); ) {
Realm realm = iterator.next();
String name = realm.getName();
if (name.equals("iniRealm")) {
String name = realm.getClass().getName();
if (LOG.isDebugEnabled()) {
LOG.debug("RealmClass.getName: " + name);
}
if (name.equals("org.apache.shiro.realm.text.IniRealm")) {
usersList.addAll(getUserListObj.getUserList((IniRealm) realm));
rolesList.addAll(getUserListObj.getRolesList((IniRealm) realm));
} else if (name.equals("ldapRealm")) {
} else if (name.equals("org.apache.zeppelin.realm.LdapGroupRealm")) {
usersList.addAll(getUserListObj.getUserList((JndiLdapRealm) realm, searchText));
} else if (name.equals("activeDirectoryRealm")) {
} else if (name.equals("org.apache.zeppelin.realm.LdapRealm")) {
usersList.addAll(getUserListObj.getUserList((LdapRealm) realm, searchText));
rolesList.addAll(getUserListObj.getRolesList((LdapRealm) realm));
} else if (name.equals("org.apache.zeppelin.realm.ActiveDirectoryGroupRealm")) {
usersList.addAll(getUserListObj.getUserList((ActiveDirectoryGroupRealm) realm,
searchText));
} else if (name.equals("jdbcRealm")) {
} else if (name.equals("org.apache.shiro.realm.jdbc.JdbcRealm")) {
usersList.addAll(getUserListObj.getUserList((JdbcRealm) realm));
}
}

51
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -187,7 +187,8 @@ public class NotebookServer extends WebSocketServlet implements
if (StringUtils.isEmpty(conn.getUser())) {
addUserConnection(messagereceived.principal, conn);
}
AuthenticationInfo subject = new AuthenticationInfo(messagereceived.principal);
AuthenticationInfo subject =
new AuthenticationInfo(messagereceived.principal, messagereceived.ticket);
/** Lets be elegant here */
switch (messagereceived.op) {
@ -565,6 +566,20 @@ public class NotebookServer extends WebSocketServlet implements
broadcast(noteId, new Message(OP.INTERPRETER_BINDINGS).put("interpreterBindings", settingList));
}
public void broadcastParagraph(Note note, Paragraph p) {
broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p));
}
private void broadcastNewParagraph(Note note, Paragraph para) {
LOG.info("Broadcasting paragraph on run call instead of note.");
int paraIndex = note.getParagraphs().indexOf(para);
broadcast(
note.getId(),
new Message(OP.PARAGRAPH_ADDED)
.put("paragraph", para)
.put("index", paraIndex));
}
public void broadcastNoteList(AuthenticationInfo subject, HashSet userAndRoles) {
if (subject == null) {
subject = new AuthenticationInfo(StringUtils.EMPTY);
@ -712,7 +727,10 @@ public class NotebookServer extends WebSocketServlet implements
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
note.persist(subject);
broadcastNote(note);
broadcast(note.getId(), new Message(OP.NOTE_UPDATED)
.put("name", name)
.put("config", config)
.put("info", note.getInfo()));
broadcastNoteList(subject, userAndRoles);
}
}
@ -854,7 +872,7 @@ public class NotebookServer extends WebSocketServlet implements
p.setTitle((String) fromMessage.get("title"));
p.setText((String) fromMessage.get("paragraph"));
note.persist(subject);
broadcast(note.getId(), new Message(OP.PARAGRAPH).put("paragraph", p));
broadcastParagraph(note, p);;
}
private void cloneNote(NotebookSocket conn, HashSet<String> userAndRoles,
@ -927,9 +945,12 @@ public class NotebookServer extends WebSocketServlet implements
/** We dont want to remove the last paragraph */
if (!note.isLastParagraph(paragraphId)) {
note.removeParagraph(subject.getUser(), paragraphId);
Paragraph para = note.removeParagraph(subject.getUser(), paragraphId);
note.persist(subject);
broadcastNote(note);
if (para != null) {
broadcast(note.getId(), new Message(OP.PARAGRAPH_REMOVED).
put("id", para.getId()));
}
}
}
@ -947,9 +968,9 @@ public class NotebookServer extends WebSocketServlet implements
userAndRoles, notebookAuthorization.getWriters(noteId));
return;
}
note.clearParagraphOutput(paragraphId);
broadcastNote(note);
Paragraph paragraph = note.getParagraph(paragraphId);
broadcastParagraph(note, paragraph);
}
private void completion(NotebookSocket conn, HashSet<String> userAndRoles, Notebook notebook,
@ -1234,7 +1255,9 @@ public class NotebookServer extends WebSocketServlet implements
note.moveParagraph(paragraphId, newIndex);
note.persist(subject);
broadcastNote(note);
broadcast(note.getId(), new Message(OP.PARAGRAPH_MOVED)
.put("id", paragraphId)
.put("index", newIndex));
}
private void insertParagraph(NotebookSocket conn, HashSet<String> userAndRoles,
@ -1251,9 +1274,9 @@ public class NotebookServer extends WebSocketServlet implements
return;
}
note.insertParagraph(index);
Paragraph newPara = note.insertParagraph(index);
note.persist(subject);
broadcastNote(note);
broadcastNewParagraph(note, newPara);
}
private void cancelParagraph(NotebookSocket conn, HashSet<String> userAndRoles, Notebook notebook,
@ -1310,7 +1333,8 @@ public class NotebookServer extends WebSocketServlet implements
boolean isTheLastParagraph = note.isLastParagraph(p.getId());
if (!(text.trim().equals(p.getMagic()) || Strings.isNullOrEmpty(text)) &&
isTheLastParagraph) {
note.addParagraph();
Paragraph newPara = note.addParagraph();
broadcastNewParagraph(note, newPara);
}
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
@ -1638,8 +1662,9 @@ public class NotebookServer extends WebSocketServlet implements
LOG.error(e.toString(), e);
}
}
notebookServer.broadcastNote(note);
if (job instanceof Paragraph) {
notebookServer.broadcastParagraph(note, (Paragraph) job);
}
try {
notebookServer.broadcastUpdateNoteJobInfo(System.currentTimeMillis() - 5000);
} catch (IOException e) {

13
zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
View file

@ -34,6 +34,10 @@ import org.apache.shiro.subject.Subject;
import org.apache.shiro.util.ThreadContext;
import org.apache.shiro.web.mgt.DefaultWebSecurityManager;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.realm.LdapRealm;
import org.mortbay.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
@ -45,6 +49,7 @@ public class SecurityUtils {
private static final String ANONYMOUS = "anonymous";
private static final HashSet<String> EMPTY_HASHSET = Sets.newHashSet();
private static boolean isEnabled = false;
private static final Logger log = LoggerFactory.getLogger(SecurityUtils.class);
public static void initSecurityManager(String shiroPath) {
IniSecurityManagerFactory factory = new IniSecurityManagerFactory("file:" + shiroPath);
@ -119,13 +124,15 @@ public class SecurityUtils {
Collection realmsList = SecurityUtils.getRealmsList();
for (Iterator<Realm> iterator = realmsList.iterator(); iterator.hasNext(); ) {
Realm realm = iterator.next();
String name = realm.getName();
if (name.equals("iniRealm")) {
String name = realm.getClass().getName();
if (name.equals("org.apache.shiro.realm.text.IniRealm")) {
allRoles = ((IniRealm) realm).getIni().get("roles");
break;
} else if (name.equals("org.apache.zeppelin.realm.LdapRealm")) {
allRoles = ((LdapRealm) realm).getListRoles();
break;
}
}
if (allRoles != null) {
Iterator it = allRoles.entrySet().iterator();
while (it.hasNext()) {

5
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -440,8 +440,9 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
waitForParagraph(1, "READY");
driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")).sendKeys(Keys.SHIFT + "5");
driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")).sendKeys("md" + Keys.ENTER);
setTextOfParagraph(1, "%md");
driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")).sendKeys(Keys.ARROW_RIGHT);
driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")).sendKeys(Keys.ENTER);
driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")).sendKeys(Keys.SHIFT + "3");
driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea")).sendKeys(" abc");

56
zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java
View file

@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory;
public class SparkParagraphIT extends AbstractZeppelinIT {
private static final Logger LOG = LoggerFactory.getLogger(SparkParagraphIT.class);
@Rule
public ErrorCollector collector = new ErrorCollector();
@ -142,8 +141,8 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
// the last statement's evaluation result is printed
setTextOfParagraph(2, "%pyspark\\n" +
"sc.version\\n" +
"1+1");
"sc.version\\n" +
"1+1");
runParagraph(2);
try {
waitForParagraph(2, "FINISHED");
@ -156,7 +155,7 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
WebElement paragraph2Result = driver.findElement(By.xpath(
getParagraphXPath(2) + "//div[contains(@id,\"_text\")]"));
collector.checkThat("Paragraph from SparkParagraphIT of testPySpark result: ",
paragraph2Result.getText().toString(), CoreMatchers.equalTo("2")
paragraph2Result.getText().toString(), CoreMatchers.equalTo("2")
);
} catch (Exception e) {
@ -192,4 +191,53 @@ public class SparkParagraphIT extends AbstractZeppelinIT {
handleException("Exception in SparkParagraphIT while testSqlSpark", e);
}
}
@Test
public void testDep() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// restart spark interpreter before running %dep
clickAndWait(By.xpath("//span[@tooltip='Interpreter binding']"));
clickAndWait(By.xpath("//div[font[contains(text(), 'spark')]]/preceding-sibling::a[@tooltip='Restart']"));
clickAndWait(By.xpath("//button[contains(.,'OK')]"));
setTextOfParagraph(1,"%dep z.load(\"org.apache.commons:commons-csv:1.1\")");
runParagraph(1);
try {
waitForParagraph(1, "FINISHED");
WebElement paragraph1Result = driver.findElement(By.xpath(getParagraphXPath(1) +
"//div[@class='text']"));
collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ",
paragraph1Result.getText(), CoreMatchers.containsString("res0: org.apache.zeppelin.dep.Dependency = org.apache.zeppelin.dep.Dependency"));
setTextOfParagraph(2, "import org.apache.commons.csv.CSVFormat");
runParagraph(2);
try {
waitForParagraph(2, "FINISHED");
WebElement paragraph2Result = driver.findElement(By.xpath(getParagraphXPath(2) +
"//div[@class='text']"));
collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ",
paragraph2Result.getText(), CoreMatchers.equalTo("import org.apache.commons.csv.CSVFormat"));
} catch (TimeoutException e) {
waitForParagraph(2, "ERROR");
collector.checkThat("Second paragraph from SparkParagraphIT of testDep status: ",
"ERROR", CoreMatchers.equalTo("FINISHED")
);
}
} catch (TimeoutException e) {
waitForParagraph(1, "ERROR");
collector.checkThat("First paragraph from SparkParagraphIT of testDep status: ",
"ERROR", CoreMatchers.equalTo("FINISHED")
);
}
} catch (Exception e) {
handleException("Exception in SparkParagraphIT while testDep", e);
}
}
}

185
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -18,9 +18,14 @@
package org.apache.zeppelin.rest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
@ -40,9 +45,9 @@ import org.junit.Test;
import org.junit.runners.MethodSorters;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import static org.junit.Assert.*;
import static org.hamcrest.MatcherAssert.assertThat;
/**
* Zeppelin interpreter rest api tests
@ -71,13 +76,12 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
public void getAvailableInterpreters() throws IOException {
// when
GetMethod get = httpGet("/interpreter");
JsonObject body = getBodyFieldFromResponse(get.getResponseBodyAsString());
// then
assertThat(get, isAllowed());
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, Object> body = (Map<String, Object>) resp.get("body");
assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getAvailableInterpreterSettings().size(), body.size());
assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getAvailableInterpreterSettings().size(),
body.entrySet().size());
get.releaseConnection();
}
@ -85,49 +89,131 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
public void getSettings() throws IOException {
// when
GetMethod get = httpGet("/interpreter/setting");
// then
assertThat(get, isAllowed());
// DO NOT REMOVE: implies that body is properly parsed as an array
JsonArray body = getArrayBodyFieldFromResponse(get.getResponseBodyAsString());
get.releaseConnection();
}
@Test
public void testGetNonExistInterpreterSetting() throws IOException {
// when
String nonExistInterpreterSettingId = "apache_.zeppelin_1s_.aw3some$";
GetMethod get = httpGet("/interpreter/setting/" + nonExistInterpreterSettingId);
// then
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
assertThat(get, isAllowed());
assertThat("Test get method:", get, isNotFound());
get.releaseConnection();
}
@Test
public void testSettingsCRUD() throws IOException {
// Call Create Setting REST API
String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
// when: call create setting API
String rawRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
"\"option\": { \"remote\": true, \"session\": false }}";
PostMethod post = httpPost("/interpreter/setting/", jsonRequest);
JsonObject jsonRequest = gson.fromJson(rawRequest, JsonElement.class).getAsJsonObject();
PostMethod post = httpPost("/interpreter/setting/", jsonRequest.toString());
String postResponse = post.getResponseBodyAsString();
LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
InterpreterSetting created = convertResponseToInterpreterSetting(postResponse);
String newSettingId = created.getId();
// then : call create setting API
assertThat("test create method:", post, isCreated());
Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, Object> body = (Map<String, Object>) resp.get("body");
//extract id from body string {id=2AWMQDNX7, name=md2, group=md,
String newSettingId = body.toString().split(",")[0].split("=")[1];
post.releaseConnection();
// Call Update Setting REST API
jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
"\"option\": { \"remote\": true, \"session\": false }}";
PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest);
// when: call read setting API
GetMethod get = httpGet("/interpreter/setting/" + newSettingId);
String getResponse = get.getResponseBodyAsString();
LOG.info("testSettingCRUD get response\n" + getResponse);
InterpreterSetting previouslyCreated = convertResponseToInterpreterSetting(getResponse);
// then : read Setting API
assertThat("Test get method:", get, isAllowed());
assertEquals(newSettingId, previouslyCreated.getId());
get.releaseConnection();
// when: call update setting API
jsonRequest.getAsJsonObject("properties").addProperty("propname2", "this is new prop");
PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest.toString());
LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString());
// then: call update setting API
assertThat("test update method:", put, isAllowed());
put.releaseConnection();
// Call Delete Setting REST API
// when: call delete setting API
DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId);
LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString());
// then: call delete setting API
assertThat("Test delete method:", delete, isAllowed());
delete.releaseConnection();
}
@Test
public void testCreatedInterpreterDependencies() throws IOException {
// when: Create 2 interpreter settings `md1` and `md2` which have different dep.
String md1Name = "md1";
String md2Name = "md2";
String md1Dep = "org.apache.drill.exec:drill-jdbc:jar:1.7.0";
String md2Dep = "org.apache.drill.exec:drill-jdbc:jar:1.6.0";
String reqBody1 = "{\"name\":\"" + md1Name + "\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[ {\n" +
" \"groupArtifactVersion\": \"" + md1Dep + "\",\n" +
" \"exclusions\":[]\n" +
" }]," +
"\"option\": { \"remote\": true, \"session\": false }}";
PostMethod post = httpPost("/interpreter/setting", reqBody1);
assertThat("test create method:", post, isCreated());
post.releaseConnection();
String reqBody2 = "{\"name\":\"" + md2Name + "\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[ {\n" +
" \"groupArtifactVersion\": \"" + md2Dep + "\",\n" +
" \"exclusions\":[]\n" +
" }]," +
"\"option\": { \"remote\": true, \"session\": false }}";
post = httpPost("/interpreter/setting", reqBody2);
assertThat("test create method:", post, isCreated());
post.releaseConnection();
// 1. Call settings API
GetMethod get = httpGet("/interpreter/setting");
String rawResponse = get.getResponseBodyAsString();
get.releaseConnection();
// 2. Parsing to List<InterpreterSettings>
JsonObject responseJson = gson.fromJson(rawResponse, JsonElement.class).getAsJsonObject();
JsonArray bodyArr = responseJson.getAsJsonArray("body");
List<InterpreterSetting> settings = new Gson().fromJson(bodyArr,
new TypeToken<ArrayList<InterpreterSetting>>() {
}.getType());
// 3. Filter interpreters out we have just created
InterpreterSetting md1 = null;
InterpreterSetting md2 = null;
for (InterpreterSetting setting : settings) {
if (md1Name.equals(setting.getName())) {
md1 = setting;
} else if (md2Name.equals(setting.getName())) {
md2 = setting;
}
}
// then: should get created interpreters which have different dependencies
// 4. Validate each md interpreter has its own dependencies
assertEquals(1, md1.getDependencies().size());
assertEquals(1, md2.getDependencies().size());
assertEquals(md1Dep, md1.getDependencies().get(0).getGroupArtifactVersion());
assertEquals(md2Dep, md2.getDependencies().get(0).getGroupArtifactVersion());
}
@Test
public void testSettingsCreateWithEmptyJson() throws IOException {
// Call Create Setting REST API
@ -139,33 +225,29 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testInterpreterAutoBinding() throws IOException {
// create note
// when
Note note = ZeppelinServer.notebook.createNote(anonymous);
// check interpreter is binded
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId());
assertThat(get, isAllowed());
get.addRequestHeader("Origin", "http://localhost");
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body");
assertTrue(0 < body.size());
JsonArray body = getArrayBodyFieldFromResponse(get.getResponseBodyAsString());
// then: check interpreter is binded
assertTrue(0 < body.size());
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testInterpreterRestart() throws IOException, InterruptedException {
// create new note
// when: create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph
// when: run markdown paragraph
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
@ -175,10 +257,10 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
}
assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown"));
// restart interpreter
// when: restart interpreter
for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
// Call Restart Interpreter REST API
// call restart interpreter API
PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), "");
assertThat("test interpreter restart:", put, isAllowed());
put.releaseConnection();
@ -186,7 +268,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
}
}
// run markdown paragraph, again
// when: run markdown paragraph, again
p = note.addParagraph();
p.setConfig(config);
p.setText("%md markdown restarted");
@ -195,21 +277,22 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown restarted"));
//cleanup
// then
assertEquals(p.getResult().message(), getSimulatedMarkdownResult("markdown restarted"));
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
// create new note
// when: create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph.
// when: run markdown paragraph.
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
@ -219,7 +302,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
}
assertEquals(p.getResult().message().get(0).getData(), getSimulatedMarkdownResult("markdown"));
// get md interpreter
// when: get md interpreter
InterpreterSetting mdIntpSetting = null;
for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
@ -260,7 +343,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testAddDeleteRepository() throws IOException {
// Call create repository REST API
// Call create repository API
String repoId = "securecentral";
String jsonRequest = "{\"id\":\"" + repoId +
"\",\"url\":\"https://repo1.maven.org/maven2\",\"snapshot\":\"false\"}";
@ -269,12 +352,26 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
assertThat("Test create method:", post, isCreated());
post.releaseConnection();
// Call delete repository REST API
// Call delete repository API
DeleteMethod delete = httpDelete("/interpreter/repository/" + repoId);
assertThat("Test delete method:", delete, isAllowed());
delete.releaseConnection();
}
public JsonObject getBodyFieldFromResponse(String rawResponse) {
JsonObject response = gson.fromJson(rawResponse, JsonElement.class).getAsJsonObject();
return response.getAsJsonObject("body");
}
public JsonArray getArrayBodyFieldFromResponse(String rawResponse) {
JsonObject response = gson.fromJson(rawResponse, JsonElement.class).getAsJsonObject();
return response.getAsJsonArray("body");
}
public InterpreterSetting convertResponseToInterpreterSetting(String rawResponse) {
return gson.fromJson(getBodyFieldFromResponse(rawResponse), InterpreterSetting.class);
}
public static String getSimulatedMarkdownResult(String markdown) {
return String.format("<div class=\"markdown-body\">\n<p>%s</p>\n</div>", markdown);
}

8
zeppelin-web/.eslintrc
View file

@ -2,7 +2,13 @@
"env": {
"browser": true,
"jasmine": true,
"node": true
"node": true,
"es6": true
},
"parserOptions": {
"ecmaFeatures": {
"experimentalObjectRestSpread": true
}
},
"globals": {
"angular": false,

108
zeppelin-web/Gruntfile.js
View file

@ -44,6 +44,30 @@ module.exports = function(grunt) {
// Project settings
yeoman: appConfig,
babel: {
options: {
sourceMap: true,
presets: ['es2015'],
plugins: ['transform-object-rest-spread']
},
dev: {
files: [{
expand: true,
cwd: './src/',
src: ['**/*.js'],
dest: '.tmp',
}]
},
dist: {
files: [{
expand: true,
cwd: '.tmp/concat/scripts',
src: ['scripts.js'],
dest: '.tmp/concat/scripts',
}]
}
},
// use ngAnnotate instead og ngMin
ngAnnotate: {
dist: {
@ -138,7 +162,7 @@ module.exports = function(grunt) {
'<%= yeoman.app %>/app/**/*.js',
'<%= yeoman.app %>/components/**/*.js'
],
tasks: ['newer:eslint:all', 'newer:jscs:all'],
tasks: ['newer:eslint:all', 'newer:jscs:all', 'newer:babel:dev'],
options: {
livereload: '<%= connect.options.livereload %>'
}
@ -147,11 +171,16 @@ module.exports = function(grunt) {
files: [
'<%= yeoman.app %>/**/*.html'
],
tasks: ['newer:htmlhint']
tasks: ['newer:htmlhint', 'newer:copy:dev']
},
jsTest: {
files: ['test/spec/{,*/}*.js'],
tasks: ['newer:eslint:test', 'newer:jscs:test', 'karma']
tasks: [
'newer:eslint:test',
'newer:jscs:test',
'newer:babel:dev',
'karma'
]
},
styles: {
files: [
@ -185,11 +214,12 @@ module.exports = function(grunt) {
port: 9000,
// Change this to '0.0.0.0' to access the server from outside.
hostname: 'localhost',
livereload: 35729
livereload: 35729,
base: '.tmp',
},
livereload: {
options: {
open: true,
open: false,
middleware: function(connect) {
return [
connect.static('.tmp'),
@ -197,7 +227,6 @@ module.exports = function(grunt) {
'/bower_components',
connect.static('./bower_components')
),
connect.static(appConfig.app)
];
}
}
@ -220,7 +249,7 @@ module.exports = function(grunt) {
},
dist: {
options: {
open: true,
open: false,
base: '<%= yeoman.dist %>'
}
}
@ -382,9 +411,6 @@ module.exports = function(grunt) {
}
}
},
// concat: {
// dist: {}
// },
svgmin: {
dist: {
@ -421,6 +447,60 @@ module.exports = function(grunt) {
// Copies remaining files to places other tasks can use
copy: {
dev: {
files: [{
expand: true,
dot: true,
cwd: '<%= yeoman.app %>',
dest: '.tmp',
src: [
'*.{ico,png,txt}',
'.htaccess',
'*.html',
'**/*.css',
'assets/styles/**/*',
'assets/images/**/*',
'WEB-INF/*'
]
}, {
// copy fonts
expand: true,
cwd: '<%= yeoman.app %>',
dest: '.tmp',
src: ['fonts/**/*.{eot,svg,ttf,woff}']
}, {
expand: true,
cwd: '<%= yeoman.app %>',
dest: '.tmp',
src: ['app/**/*.html', 'components/**/*.html']
}, {
expand: true,
cwd: 'bower_components/datatables/media/images',
src: '{,*/}*.{png,jpg,jpeg,gif}',
dest: '.tmp/images'
}, {
expand: true,
cwd: '.tmp/images',
dest: '.tmp/images',
src: ['generated/*']
}, {
expand: true,
cwd: 'bower_components/bootstrap/dist',
src: 'fonts/*',
dest: '.tmp'
}, {
expand: true,
cwd: 'bower_components/jquery-ui/themes/base/images',
src: '{,*/}*.{png,jpg,jpeg,gif}',
dest: '.tmp/styles/images'
}, {
expand: true,
cwd: 'bower_components/MathJax',
src: [
'extensions/**', 'jax/**', 'fonts/**'],
dest: '.tmp'
}]
},
dist: {
files: [{
expand: true,
@ -486,10 +566,10 @@ module.exports = function(grunt) {
// Run some tasks in parallel to speed up the build process
concurrent: {
server: [
'copy:styles'
'copy:dev'
],
test: [
'copy:styles'
'copy:dev',
],
dist: [
'copy:styles',
@ -516,6 +596,7 @@ module.exports = function(grunt) {
'wiredep',
'concurrent:server',
'postcss',
'babel:dev',
'connect:livereload',
'watch'
]);
@ -528,9 +609,11 @@ module.exports = function(grunt) {
grunt.registerTask('test', [
'clean:server',
'babel',
'wiredep',
'concurrent:test',
'postcss',
'babel:dev',
'connect:test',
'karma'
]);
@ -546,6 +629,7 @@ module.exports = function(grunt) {
'concurrent:dist',
'postcss',
'concat',
'babel:dist',
'ngAnnotate',
'copy:dist',
'cssmin',

50
zeppelin-web/README.md
View file

@ -1,51 +1,57 @@
# Zeppelin Web Application
This is Zeppelin's frontend project.
## Development Guide
## Compile Zeppelin web
### Packaging
### New environment
If you want to package the zeppelin-web only, simply run this command in this folder.
This will download all the dependencies including node (the binaries in the folder `zeppelin-web/node`)
If you want to compile the WebApplication only, you will have to simply run `mvn package` in this folder.
```
$ mvn package
```
This will Download all the dependencies including node js and npm (you will find the binaries in the folder `zeppelin-web/node`).
### Local Development
We are supposed to provide some **helper script** for __bower__ and __grunt__, but they are currently outdated, so you might want install them on your machine and use them instead.
It is recommended to install node 6.0.0+ since Zeppelin uses 6.9.1+ (see [creationix/nvm](https://github.com/creationix/nvm))
### Configured environment
All build commands are described in [package.json](./package.json)
Here are the basic commands to compile the WebApplication with a configured environment (Installed grunt, bower, npm)
```sh
# install required depepdencies and bower packages (only once)
$ npm install
**Build the application for production**
# build zeppelin-web for production
$ npm run build
`./grunt build`
# run frontend application only in dev mode (localhost:9000)
# you need to run zeppelin backend instance also
$ npm run start
**Run the application in dev mode**
# execute tests
$ npm run test
```
``./grunt serve``
## Troubleshooting
This will launch a Zeppelin WebApplication on port **9000** that will update on code changes.
(You will need to have Zeppelin running on the side)
#### Troubleshooting
**git error**
#### Git error
In case of the error `ECMDERR Failed to execute "git ls-remote --tags --heads git://xxxxx", exit code of #128`
change your git config with `git config --global url."https://".insteadOf git://`
**proxy issues**
#### Proxy issues
Try to add to the `.bowerrc` file the following content:
```
"proxy" : "http://<host>:<port>",
"https-proxy" : "http://<host>:<port>"
```
```
also try to add proxy info to npm install command:
```
```xml
<execution>
<id>npm install</id>
<goals>
@ -57,8 +63,8 @@ also try to add proxy info to npm install command:
</execution>
```
and retry to build again.
## Contribute on Zeppelin Web
If you wish to help us and contribute to Zeppelin WebApplication, please look at the overall project [contribution guidelines](https://zeppelin.apache.org/contribution/contributions.html) and the more focused [Zeppelin WebApplication's documentation](https://zeppelin.apache.org/contribution/webapplication.html).

17
zeppelin-web/bower
View file

@ -1,17 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"node/node" "./node_modules/bower/bin/bower" "$@"

18
zeppelin-web/grunt
View file

@ -1,18 +0,0 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"node/node" "./node_modules/.bin/grunt" "$@"

23
zeppelin-web/package.json
View file

@ -2,14 +2,24 @@
"name": "zeppelin-web",
"license": "Apache-2.0",
"version": "0.0.0",
"engines" : { "node" : ">=6.0.0" },
"scripts": {
"postinstall": "node_modules/.bin/bower install --silent",
"build": "./node_modules/.bin/grunt build",
"start": "./node_modules/.bin/grunt serve",
"test": "./node_modules/.bin/grunt test"
},
"dependencies": {
"grunt-angular-templates": "^0.5.7",
"grunt-dom-munger": "^3.4.0"
},
"devDependencies": {
"autoprefixer": "^6.1.0",
"bower": "1.7.2",
"bower": "^1.8.0",
"babel-plugin-transform-object-rest-spread": "^6.16.0",
"babel-preset-es2015": "^6.18.0",
"grunt": "^0.4.1",
"grunt-babel": "^6.0.0",
"grunt-cache-bust": "1.3.0",
"grunt-cli": "^0.1.13",
"grunt-concurrent": "^0.5.0",
@ -26,17 +36,18 @@
"grunt-goog-webfont-dl": "^0.1.2",
"grunt-htmlhint": "^0.9.13",
"grunt-jscs": "^2.1.0",
"grunt-karma": "~0.8.3",
"grunt-karma": "~2.0.0",
"grunt-newer": "^0.7.0",
"grunt-ng-annotate": "^0.10.0",
"grunt-postcss": "^0.7.1",
"grunt-svgmin": "^0.4.0",
"grunt-usemin": "^2.1.1",
"grunt-wiredep": "~2.0.0",
"karma": "~0.12.23",
"karma-coverage": "^0.5.1",
"karma-jasmine": "~0.1.5",
"karma-phantomjs-launcher": "~0.1.4",
"jasmine-core": "^2.5.2",
"karma": "~1.3.0",
"karma-coverage": "^1.1.1",
"karma-jasmine": "~1.0.2",
"karma-phantomjs-launcher": "~1.0.2",
"load-grunt-tasks": "^0.4.0",
"time-grunt": "^0.3.1"
},

27
zeppelin-web/pom.xml
View file

@ -90,7 +90,7 @@
<plugin>
<groupId>com.github.eirslett</groupId>
<artifactId>frontend-maven-plugin</artifactId>
<version>0.0.25</version>
<version>1.3</version>
<executions>
<execution>
@ -99,8 +99,8 @@
<goal>install-node-and-npm</goal>
</goals>
<configuration>
<nodeVersion>v0.12.13</nodeVersion>
<npmVersion>2.15.0</npmVersion>
<nodeVersion>v6.9.1</nodeVersion>
<npmVersion>3.10.8</npmVersion>
</configuration>
</execution>
@ -109,36 +109,29 @@
<goals>
<goal>npm</goal>
</goals>
</execution>
<execution>
<id>bower install</id>
<goals>
<goal>bower</goal>
</goals>
<configuration>
<arguments>--allow-root install</arguments>
<arguments>install</arguments>
</configuration>
</execution>
<execution>
<id>grunt build</id>
<id>npm build</id>
<goals>
<goal>grunt</goal>
<goal>npm</goal>
</goals>
<configuration>
<arguments>build</arguments>
<arguments>run build</arguments>
</configuration>
</execution>
<execution>
<id>grunt test</id>
<id>npm test</id>
<goals>
<goal>grunt</goal>
<goal>npm</goal>
</goals>
<phase>test</phase>
<configuration>
<arguments>test</arguments>
<arguments>run test</arguments>
</configuration>
</execution>

29
zeppelin-web/src/app/credential/credential.controller.js
View file

@ -24,6 +24,7 @@
$scope.credentialInfo = [];
$scope.showAddNewCredentialInfo = false;
$scope.availableInterpreters = [];
var getCredentialInfo = function() {
$http.get(baseUrlSrv.getRestApiBase() + '/credential').
@ -87,6 +88,33 @@
});
};
var getAvailableInterpreters = function() {
$http.get(baseUrlSrv.getRestApiBase() + '/interpreter/setting')
.success(function(data, status, headers, config) {
for (var setting = 0; setting < data.body.length; setting++) {
$scope.availableInterpreters.push(
data.body[setting].group + '.' + data.body[setting].name);
}
angular.element('#entityname').autocomplete({
source: $scope.availableInterpreters,
select: function(event, selected) {
$scope.entity = selected.item.value;
return false;
}
});
}).error(function(data, status, headers, config) {
console.log('Error %o %o', status, data.message);
});
};
$scope.toggleAddNewCredentialInfo = function() {
if ($scope.showAddNewCredentialInfo) {
$scope.showAddNewCredentialInfo = false;
} else {
$scope.showAddNewCredentialInfo = true;
}
};
$scope.cancelCredentialInfo = function() {
$scope.showAddNewCredentialInfo = false;
resetCredentialInfo();
@ -155,6 +183,7 @@
};
var init = function() {
getAvailableInterpreters();
getCredentialInfo();
};

4
zeppelin-web/src/app/credential/credential.html
View file

@ -26,7 +26,7 @@ limitations under the License.
<i class="icon-question" ng-style="{color: showRepositoryInfo ? '#3071A9' : 'black' }"></i>
</a>
<button class="btn btn-default btn-sm"
ng-click="showAddNewCredentialInfo = !showAddNewCredentialInfo">
ng-click="toggleAddNewCredentialInfo()">
<i class="fa fa-plus"></i>
Add
</button>
@ -59,7 +59,7 @@ limitations under the License.
</thead>
<tr>
<td>
<textarea msd-elastic ng-model="entity"></textarea>
<input id="entityname" ng-model="entity" placeholder="[Interpreter Group].[Interpreter Name]"/>
</td>
<td>
<textarea msd-elastic ng-model="username"></textarea>

2
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -90,7 +90,7 @@ limitations under the License.
</div>
<div class="box width-full"
ng-repeat="setting in interpreterSettings | orderBy: 'name' | filter: searchInterpreter" interpreter-directive>
ng-repeat="setting in interpreterSettings | orderBy: 'name' | filter: {name:searchInterpreter} " interpreter-directive>
<div id="{{setting.name | lowercase}}">
<div class="row interpreter">

122
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -362,87 +362,51 @@
return noteCopy;
};
var updateNote = function(note) {
/** update Note name */
if (note.name !== $scope.note.name) {
console.log('change note name: %o to %o', $scope.note.name, note.name);
$scope.note.name = note.name;
}
$scope.note.config = note.config;
$scope.note.info = note.info;
var newParagraphIds = note.paragraphs.map(function(x) {return x.id;});
var oldParagraphIds = $scope.note.paragraphs.map(function(x) {return x.id;});
var numNewParagraphs = newParagraphIds.length;
var numOldParagraphs = oldParagraphIds.length;
var paragraphToBeFocused;
var focusedParagraph;
for (var i = 0; i < $scope.note.paragraphs.length; i++) {
var paragraphId = $scope.note.paragraphs[i].id;
if (angular.element('#' + paragraphId + '_paragraphColumn_main').scope().paragraphFocused) {
focusedParagraph = paragraphId;
break;
var addPara = function(paragraph, index) {
$scope.note.paragraphs.splice(index, 0, paragraph);
_.each($scope.note.paragraphs, function(para) {
if (para.id === paragraph.id) {
para.focus = true;
}
}
/** add a new paragraph */
if (numNewParagraphs > numOldParagraphs) {
for (var index in newParagraphIds) {
if (oldParagraphIds[index] !== newParagraphIds[index]) {
$scope.note.paragraphs.splice(index, 0, note.paragraphs[index]);
paragraphToBeFocused = note.paragraphs[index].id;
break;
}
$scope.$broadcast('updateParagraph', {
note: $scope.note, // pass the note object to paragraph scope
paragraph: note.paragraphs[index]});
}
}
/** update or move paragraph */
if (numNewParagraphs === numOldParagraphs) {
for (var idx in newParagraphIds) {
var newEntry = note.paragraphs[idx];
if (oldParagraphIds[idx] === newParagraphIds[idx]) {
$scope.$broadcast('updateParagraph', {
note: $scope.note, // pass the note object to paragraph scope
paragraph: newEntry});
} else {
// move paragraph
var oldIdx = oldParagraphIds.indexOf(newParagraphIds[idx]);
$scope.note.paragraphs.splice(oldIdx, 1);
$scope.note.paragraphs.splice(idx, 0, newEntry);
// rebuild id list since paragraph has moved.
oldParagraphIds = $scope.note.paragraphs.map(function(x) {return x.id;});
}
if (focusedParagraph === newParagraphIds[idx]) {
paragraphToBeFocused = focusedParagraph;
}
}
}
/** remove paragraph */
if (numNewParagraphs < numOldParagraphs) {
for (var oldidx in oldParagraphIds) {
if (oldParagraphIds[oldidx] !== newParagraphIds[oldidx]) {
$scope.note.paragraphs.splice(oldidx, 1);
break;
}
}
}
// restore focus of paragraph
for (var f = 0; f < $scope.note.paragraphs.length; f++) {
if (paragraphToBeFocused === $scope.note.paragraphs[f].id) {
$scope.note.paragraphs[f].focus = true;
}
}
});
};
var removePara = function(paragraphId) {
var removeIdx;
_.each($scope.note.paragraphs, function(para, idx) {
if (para.id === paragraphId) {
removeIdx = idx;
}
});
return $scope.note.paragraphs.splice(removeIdx, 1);
};
$scope.$on('addParagraph', function(event, paragraph, index) {
addPara(paragraph, index);
});
$scope.$on('removeParagraph', function(event, paragraphId) {
removePara(paragraphId);
});
$scope.$on('moveParagraph', function(event, paragraphId, newIdx) {
var removedPara = removePara(paragraphId);
if (removedPara && removedPara.length === 1) {
addPara(removedPara[0], newIdx);
}
});
$scope.$on('updateNote', function(event, name, config, info) {
/** update Note name */
if (name !== $scope.note.name) {
console.log('change note name to : %o', $scope.note.name);
$scope.note.name = name;
}
$scope.note.config = config;
$scope.note.info = info;
initializeLookAndFeel();
});
var getInterpreterBindings = function() {
websocketMsgSrv.getInterpreterBindings($scope.note.id);
};
@ -858,8 +822,6 @@
if ($scope.note === null) {
$scope.note = note;
} else {
updateNote(note);
}
initializeLookAndFeel();
//open interpreter binding setting when there're none selected

4
zeppelin-web/src/components/navbar/navbar-noteList-elem.html
View file

@ -12,11 +12,11 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
<a class="notebook-list-item" ng-if="note.id" href="#/notebook/{{note.id}}">
<a class="notebook-list-item" ng-if="!note.children" href="#/notebook/{{note.id}}">
<i style="font-size: 10px; margin-right: 5px;" class="icon-doc"></i>
<span>{{noteName(note)}}</span>
</a>
<li ng-if="!note.id" ng-click="$event.stopPropagation()">
<li ng-if="note.children" ng-click="$event.stopPropagation()">
<expand-collapse>
<div>
<a class="notebook-list-item" href="javascript:void(0)">

8
zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js
View file

@ -140,6 +140,14 @@
$rootScope.$broadcast('configurationsInfo', data);
} else if (op === 'INTERPRETER_SETTINGS') {
$rootScope.$broadcast('interpreterSettings', data);
} else if (op === 'PARAGRAPH_ADDED') {
$rootScope.$broadcast('addParagraph', data.paragraph, data.index);
} else if (op === 'PARAGRAPH_REMOVED') {
$rootScope.$broadcast('removeParagraph', data.id);
} else if (op === 'PARAGRAPH_MOVED') {
$rootScope.$broadcast('moveParagraph', data.id, data.index);
} else if (op === 'NOTE_UPDATED') {
$rootScope.$broadcast('updateNote', data.name, data.config, data.info);
}
});

12
zeppelin-web/test/karma.conf.js
View file

@ -68,17 +68,17 @@ module.exports = function(config) {
'bower_components/MathJax/MathJax.js',
'bower_components/angular-mocks/angular-mocks.js',
// endbower
'src/app/app.js',
'src/app/app.controller.js',
'src/app/tabledata/transformation.js',
'src/app/**/*.js',
'src/components/**/*.js',
'.tmp/app/app.js',
'.tmp/app/app.controller.js',
'.tmp/app/tabledata/transformation.js',
'.tmp/app/**/*.js',
'.tmp/components/**/*.js',
'test/spec/**/*.js'
],
// list of files / patterns to exclude
exclude: [
'src/app/visualization/builtins/*.js'
'.tmp/app/visualization/builtins/*.js'
],
// web server port

2
zeppelin-web/test/spec/controllers/paragraph.js
View file

@ -54,4 +54,4 @@ describe('Controller: ParagraphCtrl', function() {
it('should set default value of "paragraphFocused" as false', function() {
expect(scope.paragraphFocused).toEqual(false);
});
});
);

2
zeppelin-zengine/pom.xml
View file

@ -250,7 +250,7 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration combine.children="append">
<configuration>
<forkMode>always</forkMode>
</configuration>
</plugin>

8
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -435,6 +435,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED);
}
public boolean isNotebokPublic() {
return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC);
}
public String getConfDir() {
return getString(ConfVars.ZEPPELIN_CONF_DIR);
}
@ -533,6 +537,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
+ "org.apache.zeppelin.flink.FlinkInterpreter,"
+ "org.apache.zeppelin.python.PythonInterpreter,"
+ "org.apache.zeppelin.python.PythonInterpreterPandasSql,"
+ "org.apache.zeppelin.python.PythonCondaInterpreter,"
+ "org.apache.zeppelin.python.PythonDockerInterpreter,"
+ "org.apache.zeppelin.ignite.IgniteInterpreter,"
+ "org.apache.zeppelin.ignite.IgniteSqlInterpreter,"
+ "org.apache.zeppelin.lens.LensInterpreter,"
@ -570,6 +576,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),
ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", VFSNotebookRepo.class.getName()),
ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC("zeppelin.notebook.one.way.sync", false),
// whether by default note is public or private
ZEPPELIN_NOTEBOOK_PUBLIC("zeppelin.notebook.public", true),
ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner",
System.getProperty("os.name")
.startsWith("Windows") ? "bin/interpreter.cmd" : "bin/interpreter.sh"),

Some files were not shown because too many files have changed in this diff Show more