Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into jdbc-impersonation

Browse source
This commit is contained in:
astroshim 2016-11-05 21:10:21 +09:00
commit b55aceb8af
140 changed files with 2224 additions and 692 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -21,6 +21,7 @@ lens/lens-cli-hist.log
conf/zeppelin-env.sh
conf/zeppelin-env.cmd
conf/zeppelin-site.xml
conf/shiro.ini
conf/keystore
conf/truststore
conf/interpreter.json

14
.travis.yml
View file

@ -36,7 +36,7 @@ matrix:
include:
# Test License compliance using RAT tool
- jdk: "oraclejdk7"
env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Test all modules with spark 2.0.0 and scala 2.11
- jdk: "oraclejdk7"
@ -58,18 +58,6 @@ matrix:
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.2.2
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.1
- jdk: "oraclejdk7"
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"

7
README.md
View file

@ -2,7 +2,7 @@
**Documentation:** [User Guide](http://zeppelin.apache.org/docs/latest/index.html)<br/>
**Mailing Lists:** [User and Dev mailing list](http://zeppelin.apache.org/community.html)<br/>
**Continuous Integration:** [![Build Status](https://secure.travis-ci.org/apache/zeppelin.png?branch=master)](https://travis-ci.org/apache/zeppelin) <br/>
**Continuous Integration:** [![Build Status](https://travis-ci.org/apache/zeppelin.svg?branch=master)](https://travis-ci.org/apache/zeppelin) <br/>
**Contributing:** [Contribution Guide](https://zeppelin.apache.org/contribution/contributions.html)<br/>
**Issue Tracker:** [Jira](https://issues.apache.org/jira/browse/ZEPPELIN)<br/>
**License:** [Apache 2.0](https://github.com/apache/zeppelin/blob/master/LICENSE)
@ -128,9 +128,6 @@ Available profiles are
-Pspark-1.6
-Pspark-1.5
-Pspark-1.4
-Pspark-1.3
-Pspark-1.2
-Pspark-1.1
-Pcassandra-spark-1.5
-Pcassandra-spark-1.4
-Pcassandra-spark-1.3
@ -192,7 +189,7 @@ enable 3rd party vendor repository (cloudera)
##### `-Pmapr[version]` (optional)
For the MapR Hadoop Distribution, these profiles will handle the Hadoop version. As MapR allows different versions of Spark to be installed, you should specify which version of Spark is installed on the cluster by adding a Spark profile (`-Pspark-1.2`, `-Pspark-1.3`, etc.) as needed.
For the MapR Hadoop Distribution, these profiles will handle the Hadoop version. As MapR allows different versions of Spark to be installed, you should specify which version of Spark is installed on the cluster by adding a Spark profile (`-Pspark-1.6`, `-Pspark-2.0`, etc.) as needed.
The correct Maven artifacts can be found for every version of MapR at http://doc.mapr.com
Available profiles are

4
alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java
View file

@ -135,7 +135,7 @@ public class AlluxioInterpreter extends Interpreter {
private String[] splitAndRemoveEmpty(String st, String splitSeparator) {
String[] voices = st.split(splitSeparator);
ArrayList<String> result = new ArrayList<String>();
ArrayList<String> result = new ArrayList<>();
for (String voice : voices) {
if (!voice.trim().isEmpty()) {
result.add(voice);
@ -145,7 +145,7 @@ public class AlluxioInterpreter extends Interpreter {
}
private String[] splitAndRemoveEmpty(String[] sts, String splitSeparator) {
ArrayList<String> result = new ArrayList<String>();
ArrayList<String> result = new ArrayList<>();
for (String st : sts) {
result.addAll(Arrays.asList(splitAndRemoveEmpty(st, splitSeparator)));
}

2
alluxio/src/test/java/org/apache/zeppelin/alluxio/AlluxioInterpreterTest.java
View file

@ -93,7 +93,7 @@ public class AlluxioInterpreterTest {
List expectedResultThree = Arrays.asList(
new InterpreterCompletion("copyFromLocal", "copyFromLocal"),
new InterpreterCompletion("copyToLocal", "copyToLocal"));
List expectedResultNone = new ArrayList<String>();
List expectedResultNone = new ArrayList<>();
List<InterpreterCompletion> resultOne = alluxioInterpreter.completion("c", 0);
List<InterpreterCompletion> resultTwo = alluxioInterpreter.completion("co", 0);

4
conf/shiro.ini → conf/shiro.ini.template
View file

@ -78,5 +78,5 @@ admin = *
#/api/interpreter/** = authc, roles[admin]
#/api/configurations/** = authc, roles[admin]
#/api/credential/** = authc, roles[admin]
/** = anon
#/** = authc
#/** = anon
/** = authc

1
docs/install/upgrade.md
View file

@ -52,3 +52,4 @@ So, copying `notebook` and `conf` directory should be enough.
- From 0.7, we don't use `ZEPPELIN_JAVA_OPTS` as default value of `ZEPPELIN_INTP_JAVA_OPTS` and also the same for `ZEPPELIN_MEM`/`ZEPPELIN_INTP_MEM`. If user want to configure the jvm opts of interpreter process, please set `ZEPPELIN_INTP_JAVA_OPTS` and `ZEPPELIN_INTP_MEM` explicitly. If you don't set `ZEPPELIN_INTP_MEM`, Zeppelin will set it to `-Xms1024m -Xmx1024m -XX:MaxPermSize=512m` by default.
- Mapping from `%jdbc(prefix)` to `%prefix` is no longer available. Instead, you can use %[interpreter alias] with multiple interpreter setttings on GUI.
- Usage of `ZEPPELIN_PORT` is not supported in ssl mode. Instead use `ZEPPELIN_SSL_PORT` to configure the ssl port. Value from `ZEPPELIN_PORT` is used only when `ZEPPELIN_SSL` is set to `false`.
- The support on Spark 1.1.x to 1.3.x is deprecated.

10
docs/rest-api/rest-interpreter.md
View file

@ -400,6 +400,16 @@ The role of registered interpreters, settings and interpreters group are describ
<td>Fail code</td>
<td> 500 </td>
</tr>
<tr>
<td>Sample JSON input (Optional)</td>
<td>
<pre>
{
"noteId": "2AVQJVC8N"
}
</pre>
</td>
</tr>
<tr>
<td>Sample JSON response</td>
<td>

38
docs/rest-api/rest-notebook.md
View file

@ -493,7 +493,7 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
<col width="200">
<tr>
<td>Description</td>
<td> This ```POST``` method runs the paragraph synchronously by given note and paragraph id. This API can return SUCCESS or ERROR depending on the outcome of the paragraph execution
<td>This ```POST``` method runs the paragraph synchronously by given note and paragraph id. This API can return SUCCESS or ERROR depending on the outcome of the paragraph execution
</td>
</tr>
<tr>
@ -972,3 +972,39 @@ If you work with Apache Zeppelin and find a need for an additional REST API, ple
</tr>
</tr>
</table>
<br />
### Clear all paragraph result
<table class="table-configuration">
<col width="200">
<tr>
<td>Description</td>
<td>This ```PUT``` method clear all paragraph results from note of given id.
</td>
</tr>
<tr>
<td>URL</td>
<td>```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[noteId]/clear```</td>
</tr>
<tr>
<td>Success code</td>
<td>200</td>
</tr>
<tr>
<td>Forbidden code</td>
<td>401</td>
</tr>
<tr>
<td>Not Found code</td>
<td>404</td>
</tr>
<tr>
<td>Fail code</td>
<td>500</td>
</tr>
<tr>
<td>sample JSON response</td>
<td><pre>{"status": "OK"}</pre></td>
</tr>
</tr>
</table>

4
docs/security/authentication.md
View file

@ -85,7 +85,7 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
}
location /ws { # For websocket support
proxy_pass http://zeppelin;
proxy_pass http://zeppelin/ws;
proxy_http_version 1.1;
proxy_set_header Upgrade websocket;
proxy_set_header Connection upgrade;
@ -130,4 +130,4 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
Another option is to have an authentication server that can verify user credentials in an LDAP server.
If an incoming request to the Zeppelin server does not have a cookie with user information encrypted with the authentication server public key, the user
is redirected to the authentication server. Once the user is verified, the authentication server redirects the browser to a specific URL in the Zeppelin server which sets the authentication cookie in the browser.
The end result is that all requests to the Zeppelin web server have the authentication cookie which contains user and groups information.
The end result is that all requests to the Zeppelin web server have the authentication cookie which contains user and groups information.

21
docs/security/shiroauthentication.md
View file

@ -31,7 +31,11 @@ When you connect to Apache Zeppelin, you will be asked to enter your credentials
## Security Setup
You can setup **Zeppelin notebook authentication** in some simple steps.
### 1. Secure the HTTP channel
### 1. Enable Shiro
By default in `conf`, you will find `shiro.ini.template`, this file is used as an example and it is strongly recommended
to create a `shiro.ini` file by doing the following command line `cp conf/shiro.ini.template conf/shiro.ini`.
### 2. Secure the HTTP channel
To secure the HTTP channel, you have to change both **anon** and **authc** settings in `conf/shiro.ini`. In here, **anon** means "the access is anonymous" and **authc** means "formed auth security".
The default status of them is
@ -49,10 +53,10 @@ Deactivate the line "/** = anon" and activate the line "/** = authc" in `conf/sh
For the further information about `shiro.ini` file format, please refer to [Shiro Configuration](http://shiro.apache.org/configuration.html#Configuration-INISections).
### 2. Secure the Websocket channel
### 3. Secure the Websocket channel
Set to property **zeppelin.anonymous.allowed** to **false** in `conf/zeppelin-site.xml`. If you don't have this file yet, just copy `conf/zeppelin-site.xml.template` to `conf/zeppelin-site.xml`.
### 3. Start Zeppelin
### 4. Start Zeppelin
```
bin/zeppelin-daemon.sh start (or restart)
@ -60,7 +64,7 @@ bin/zeppelin-daemon.sh start (or restart)
Then you can browse Zeppelin at [http://localhost:8080](http://localhost:8080).
### 4. Login
### 5. Login
Finally, you can login using one of the below **username/password** combinations.
<center><img src="../assets/themes/zeppelin/img/docs-img/zeppelin-login.png"></center>
@ -94,7 +98,7 @@ ldapRealm.contextFactory.url = ldap://ldap.test.com:389
ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
ldapRealm.contextFactory.authenticationMechanism = SIMPLE
```
also define roles/groups that you want to have in system, like below;
```
@ -159,8 +163,8 @@ securityManager.realms = $zeppelinHubRealm
> Note: ZeppelinHub is not releated to apache Zeppelin project.
## Secure your Zeppelin information (optional)
By default, anyone who defined in `[users]` can share **Interpreter Setting**, **Credential** and **Configuration** information in Apache Zeppelin.
Sometimes you might want to hide these information for your use case.
By default, anyone who defined in `[users]` can share **Interpreter Setting**, **Credential** and **Configuration** information in Apache Zeppelin.
Sometimes you might want to hide these information for your use case.
Since Shiro provides **url-based security**, you can hide the information by commenting or uncommenting these below lines in `conf/shiro.ini`.
```
@ -171,9 +175,8 @@ Since Shiro provides **url-based security**, you can hide the information by com
/api/credential/** = authc, roles[admin]
```
In this case, only who have `admin` role can see **Interpreter Setting**, **Credential** and **Configuration** information.
In this case, only who have `admin` role can see **Interpreter Setting**, **Credential** and **Configuration** information.
If you want to grant this permission to other users, you can change **roles[ ]** as you defined at `[users]` section.
<br/>
> **NOTE :** All of the above configurations are defined in the `conf/shiro.ini` file. This documentation is originally from [SECURITY-README.md](https://github.com/apache/zeppelin/blob/master/SECURITY-README.md).

2
file/src/test/java/org/apache/zeppelin/file/HDFSFileInterpreterTest.java
View file

@ -123,7 +123,7 @@ public class HDFSFileInterpreterTest extends TestCase {
* Store command results from curl against a real file system
*/
class MockFileSystem {
HashMap<String, String> mfs = new HashMap<String, String>();
HashMap<String, String> mfs = new HashMap<>();
void addListStatusData() {
mfs.put("/?op=LISTSTATUS",
"{\"FileStatuses\":{\"FileStatus\":[\n" +

6
flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java
View file

@ -175,7 +175,7 @@ public class FlinkInterpreter extends Interpreter {
pathSettings.v_$eq(classpath);
settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings);
settings.explicitParentLoader_$eq(new Some<ClassLoader>(Thread.currentThread()
settings.explicitParentLoader_$eq(new Some<>(Thread.currentThread()
.getContextClassLoader()));
BooleanSetting b = (BooleanSetting) settings.usejavacp();
b.v_$eq(true);
@ -197,7 +197,7 @@ public class FlinkInterpreter extends Interpreter {
}
private List<File> classPath(ClassLoader cl) {
List<File> paths = new LinkedList<File>();
List<File> paths = new LinkedList<>();
if (cl == null) {
return paths;
}
@ -217,7 +217,7 @@ public class FlinkInterpreter extends Interpreter {
public Object getLastObject() {
Object obj = imain.lastRequest().lineRep().call(
"$result",
JavaConversions.asScalaBuffer(new LinkedList<Object>()));
JavaConversions.asScalaBuffer(new LinkedList<>()));
return obj;
}

2
ignite/src/main/java/org/apache/zeppelin/ignite/IgniteInterpreter.java
View file

@ -178,7 +178,7 @@ public class IgniteInterpreter extends Interpreter {
public Object getLastObject() {
Object obj = imain.lastRequest().lineRep().call(
"$result",
JavaConversions.asScalaBuffer(new LinkedList<Object>()));
JavaConversions.asScalaBuffer(new LinkedList<>()));
return obj;
}

4
lens/src/main/java/org/apache/zeppelin/lens/LensInterpreter.java
View file

@ -76,9 +76,9 @@ public class LensInterpreter extends Interpreter {
private static Pattern s_queryExecutePattern = Pattern.compile(".*query\\s+execute\\s+(.*)");
private static Map<String, ExecutionDetail> s_paraToQH =
new ConcurrentHashMap<String, ExecutionDetail> (); //tracks paragraphId -> Lens QueryHandle
new ConcurrentHashMap<> (); //tracks paragraphId -> Lens QueryHandle
private static Map<LensClient, Boolean> s_clientMap =
new ConcurrentHashMap<LensClient, Boolean>();
new ConcurrentHashMap<>();
private int m_maxResults;
private int m_maxThreads;

197
livy/pom.xml
View file

@ -41,6 +41,8 @@
<assertj.version>1.7.0</assertj.version>
<mockito.version>1.9.5</mockito.version>
<livy.version>0.2.0</livy.version>
<spark.version>1.5.2</spark.version>
<hadoop.version>2.6.0</hadoop.version>
</properties>
<dependencies>
@ -111,38 +113,218 @@
<groupId>com.cloudera.livy</groupId>
<artifactId>livy-integration-test</artifactId>
<version>${livy.version}</version>
<scope>compile</scope>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-repl_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.cloudera.livy</groupId>
<artifactId>livy-test-lib</artifactId>
<version>${livy.version}</version>
<scope>compile</scope>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-repl_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.10</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.cloudera.livy</groupId>
<artifactId>livy-core</artifactId>
<version>${livy.version}</version>
<scope>compile</scope>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-repl_2.10</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.10</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-repl_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>2.6.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
<version>${hadoop.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<repositories>
@ -250,9 +432,10 @@
</goals>
<configuration>
<target>
<delete file="${project.build.directory}/unit-tests.log" quiet="true" />
<delete file="${project.build.directory}/jacoco.exec" quiet="true" />
<delete dir="${project.build.directory}/tmp" quiet="true" />
<delete file="${project.build.directory}/unit-tests.log"
quiet="true"/>
<delete file="${project.build.directory}/jacoco.exec" quiet="true"/>
<delete dir="${project.build.directory}/tmp" quiet="true"/>
</target>
</configuration>
</execution>
@ -265,7 +448,7 @@
</goals>
<configuration>
<target>
<mkdir dir="${project.build.directory}/tmp" />
<mkdir dir="${project.build.directory}/tmp"/>
</target>
</configuration>
</execution>

8
livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java
View file

@ -57,7 +57,7 @@ public class LivyHelper {
public Integer createSession(InterpreterContext context, String kind) throws Exception {
try {
Map<String, String> conf = new HashMap<String, String>();
Map<String, String> conf = new HashMap<>();
Iterator<Entry<Object, Object>> it = property.entrySet().iterator();
while (it.hasNext()) {
@ -352,16 +352,16 @@ public class LivyHelper {
ResponseEntity<String> response = null;
try {
if (method.equals("POST")) {
HttpEntity<String> entity = new HttpEntity<String>(jsonData, headers);
HttpEntity<String> entity = new HttpEntity<>(jsonData, headers);
response = restTemplate.exchange(targetURL, HttpMethod.POST, entity, String.class);
paragraphHttpMap.put(paragraphId, response);
} else if (method.equals("GET")) {
HttpEntity<String> entity = new HttpEntity<String>(headers);
HttpEntity<String> entity = new HttpEntity<>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.GET, entity, String.class);
paragraphHttpMap.put(paragraphId, response);
} else if (method.equals("DELETE")) {
HttpEntity<String> entity = new HttpEntity<String>(headers);
HttpEntity<String> entity = new HttpEntity<>(headers);
response = restTemplate.exchange(targetURL, HttpMethod.DELETE, entity, String.class);
}
} catch (HttpClientErrorException e) {

2
pig/src/main/java/org/apache/zeppelin/pig/PigQueryInterpreter.java
View file

@ -68,7 +68,7 @@ public class PigQueryInterpreter extends BasePigInterpreter {
// '-' is invalid for pig alias
String alias = "paragraph_" + context.getParagraphId().replace("-", "_");
String[] lines = st.split("\n");
List<String> queries = new ArrayList<String>();
List<String> queries = new ArrayList<>();
for (int i = 0; i < lines.length; ++i) {
if (i == lines.length - 1) {
lines[i] = alias + " = " + lines[i];

2
postgresql/src/main/java/org/apache/zeppelin/postgresql/PostgreSqlInterpreter.java
View file

@ -324,7 +324,7 @@ public class PostgreSqlInterpreter extends Interpreter {
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
List<CharSequence> candidates = new ArrayList<CharSequence>();
List<CharSequence> candidates = new ArrayList<>();
if (sqlCompleter != null && sqlCompleter.complete(buf, cursor, candidates) >= 0) {
List completion = Lists.transform(candidates, sequenceToStringTransformer);
return completion;

6
postgresql/src/main/java/org/apache/zeppelin/postgresql/SqlCompleter.java
View file

@ -65,7 +65,7 @@ public class SqlCompleter extends StringsCompleter {
}
};
private Set<String> modelCompletions = new HashSet<String>();
private Set<String> modelCompletions = new HashSet<>();
public SqlCompleter(Set<String> allCompletions, Set<String> dataModelCompletions) {
super(allCompletions);
@ -150,7 +150,7 @@ public class SqlCompleter extends StringsCompleter {
keywords += "," + driverKeywords.toUpperCase();
}
Set<String> completions = new TreeSet<String>();
Set<String> completions = new TreeSet<>();
// Add the keywords from the current JDBC connection
@ -193,7 +193,7 @@ public class SqlCompleter extends StringsCompleter {
public static Set<String> getDataModelMetadataCompletions(Connection connection)
throws SQLException {
Set<String> completions = new TreeSet<String>();
Set<String> completions = new TreeSet<>();
getColumnNames(connection.getMetaData(), completions);
getSchemaNames(connection.getMetaData(), completions);
return completions;

4
postgresql/src/test/java/org/apache/zeppelin/postgresql/SqlCompleterTest.java
View file

@ -39,7 +39,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
private Logger logger = LoggerFactory.getLogger(SqlCompleterTest.class);
private final static Set<String> EMPTY = new HashSet<String>();
private final static Set<String> EMPTY = new HashSet<>();
private CompleterTester tester;
@ -157,7 +157,7 @@ public class SqlCompleterTest extends BasicJDBCTestCaseAdapter {
private void expectedCompletions(String buffer, int cursor, Set<String> expected) {
ArrayList<CharSequence> candidates = new ArrayList<CharSequence>();
ArrayList<CharSequence> candidates = new ArrayList<>();
completer.complete(buffer, cursor, candidates);

2
shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java
View file

@ -60,7 +60,7 @@ public class ShellInterpreter extends Interpreter {
@Override
public void open() {
LOGGER.info("Command timeout property: {}", getProperty(TIMEOUT_PROPERTY));
executors = new ConcurrentHashMap<String, DefaultExecutor>();
executors = new ConcurrentHashMap<>();
if (!StringUtils.isAnyEmpty(getProperty("zeppelin.shell.auth.type"))) {
ShellSecurityImpl.createSecureConfiguration(getProperty(), shell);
}

32
spark/pom.xml
View file

@ -450,38 +450,6 @@
</build>
<profiles>
<profile>
<id>spark-1.1</id>
<dependencies>
</dependencies>
<properties>
<spark.version>1.1.1</spark.version>
<akka.version>2.2.3-shaded-protobuf</akka.version>
</properties>
</profile>
<profile>
<id>spark-1.2</id>
<dependencies>
</dependencies>
<properties>
<spark.version>1.2.1</spark.version>
</properties>
</profile>
<profile>
<id>spark-1.3</id>
<properties>
<spark.version>1.3.1</spark.version>
</properties>
<dependencies>
</dependencies>
</profile>
<profile>
<id>spark-1.4</id>
<properties>

10
spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java
View file

@ -153,7 +153,7 @@ public class DepInterpreter extends Interpreter {
settings.scala$tools$nsc$settings$ScalaSettings$_setter_$classpath_$eq(pathSettings);
// set classloader for scala compiler
settings.explicitParentLoader_$eq(new Some<ClassLoader>(Thread.currentThread()
settings.explicitParentLoader_$eq(new Some<>(Thread.currentThread()
.getContextClassLoader()));
BooleanSetting b = (BooleanSetting) settings.usejavacp();
@ -219,7 +219,7 @@ public class DepInterpreter extends Interpreter {
public Object getLastObject() {
IMain.Request r = (IMain.Request) Utils.invokeMethod(intp, "lastRequest");
Object obj = r.lineRep().call("$result",
JavaConversions.asScalaBuffer(new LinkedList<Object>()));
JavaConversions.asScalaBuffer(new LinkedList<>()));
return obj;
}
@ -290,7 +290,7 @@ public class DepInterpreter extends Interpreter {
Candidates ret = c.complete(buf, cursor);
List<String> candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates());
List<InterpreterCompletion> completions = new LinkedList<InterpreterCompletion>();
List<InterpreterCompletion> completions = new LinkedList<>();
for (String candidate : candidates) {
completions.add(new InterpreterCompletion(candidate, candidate));
@ -298,7 +298,7 @@ public class DepInterpreter extends Interpreter {
return completions;
} else {
return new LinkedList<InterpreterCompletion>();
return new LinkedList<>();
}
}
@ -314,7 +314,7 @@ public class DepInterpreter extends Interpreter {
}
private List<File> classPath(ClassLoader cl) {
List<File> paths = new LinkedList<File>();
List<File> paths = new LinkedList<>();
if (cl == null) {
return paths;
}

17
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -49,9 +49,9 @@ import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.spark.dep.SparkDependencyContext;
import org.slf4j.Logger;
@ -115,7 +115,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
// load libraries from Dependency Interpreter
URL [] urls = new URL[0];
List<URL> urlList = new LinkedList<URL>();
List<URL> urlList = new LinkedList<>();
if (depInterpreter != null) {
SparkDependencyContext depc = depInterpreter.getDependencyContext();
@ -165,6 +165,15 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
}
}
private Map setupPySparkEnv() throws IOException{
Map env = EnvironmentUtils.getProcEnvironment();
if (!env.containsKey("PYTHONPATH")) {
SparkConf conf = getSparkConf();
env.put("PYTHONPATH", conf.get("spark.submit.pyFiles").replaceAll(",", ":"));
}
return env;
}
private void createGatewayServerAndStartScript() {
// create python script
createPythonScript();
@ -196,10 +205,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
executor.setStreamHandler(streamHandler);
executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
try {
Map env = EnvironmentUtils.getProcEnvironment();
Map env = setupPySparkEnv();
executor.execute(cmd, env, this);
pythonscriptRunning = true;
} catch (IOException e) {

13
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -504,6 +504,7 @@ public class SparkInterpreter extends Interpreter {
conf.set("spark.files", conf.get("spark.yarn.dist.files"));
}
conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
conf.set("spark.submit.pyFiles", Joiner.on(",").join(pythonLibUris));
}
// Distributes needed libraries to workers
@ -596,7 +597,7 @@ public class SparkInterpreter extends Interpreter {
}
String[] argsArray = args.split(" ");
LinkedList<String> argList = new LinkedList<String>();
LinkedList<String> argList = new LinkedList<>();
for (String arg : argsArray) {
argList.add(arg);
}
@ -719,7 +720,7 @@ public class SparkInterpreter extends Interpreter {
// set classloader for scala compiler
settings.explicitParentLoader_$eq(new Some<ClassLoader>(Thread.currentThread()
settings.explicitParentLoader_$eq(new Some<>(Thread.currentThread()
.getContextClassLoader()));
BooleanSetting b = (BooleanSetting) settings.usejavacp();
b.v_$eq(true);
@ -957,7 +958,7 @@ public class SparkInterpreter extends Interpreter {
}
private List<File> classPath(ClassLoader cl) {
List<File> paths = new LinkedList<File>();
List<File> paths = new LinkedList<>();
if (cl == null) {
return paths;
}
@ -978,7 +979,7 @@ public class SparkInterpreter extends Interpreter {
public List<InterpreterCompletion> completion(String buf, int cursor) {
if (completer == null) {
logger.warn("Can't find completer");
return new LinkedList<InterpreterCompletion>();
return new LinkedList<>();
}
if (buf.length() < cursor) {
@ -994,7 +995,7 @@ public class SparkInterpreter extends Interpreter {
Candidates ret = c.complete(completionText, cursor);
List<String> candidates = WrapAsJava$.MODULE$.seqAsJavaList(ret.candidates());
List<InterpreterCompletion> completions = new LinkedList<InterpreterCompletion>();
List<InterpreterCompletion> completions = new LinkedList<>();
for (String candidate : candidates) {
completions.add(new InterpreterCompletion(candidate, candidate));
@ -1067,7 +1068,7 @@ public class SparkInterpreter extends Interpreter {
return null;
}
Object obj = r.lineRep().call("$result",
JavaConversions.asScalaBuffer(new LinkedList<Object>()));
JavaConversions.asScalaBuffer(new LinkedList<>()));
return obj;
}

6
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
View file

@ -61,7 +61,7 @@ public class ZeppelinContext {
// given replName in parapgraph
private static final Map<String, String> interpreterClassMap;
static {
interpreterClassMap = new HashMap<String, String>();
interpreterClassMap = new HashMap<>();
interpreterClassMap.put("spark", "org.apache.zeppelin.spark.SparkInterpreter");
interpreterClassMap.put("sql", "org.apache.zeppelin.spark.SparkSqlInterpreter");
interpreterClassMap.put("dep", "org.apache.zeppelin.spark.DepInterpreter");
@ -134,7 +134,7 @@ public class ZeppelinContext {
@ZeppelinApi
public scala.collection.Iterable<Object> checkbox(String name,
scala.collection.Iterable<Tuple2<Object, String>> options) {
List<Object> allChecked = new LinkedList<Object>();
List<Object> allChecked = new LinkedList<>();
for (Tuple2<Object, String> option : asJavaIterable(options)) {
allChecked.add(option._1());
}
@ -400,7 +400,7 @@ public class ZeppelinContext {
@ZeppelinApi
public List<String> listParagraphs() {
List<String> paragraphs = new LinkedList<String>();
List<String> paragraphs = new LinkedList<>();
for (InterpreterContextRunner r : interpreterContext.getRunners()) {
paragraphs.add(r.getParagraphId());

18
spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java
View file

@ -49,16 +49,16 @@ import scala.Console;
*
*/
public class SparkDependencyContext {
List<Dependency> dependencies = new LinkedList<Dependency>();
List<Repository> repositories = new LinkedList<Repository>();
List<Dependency> dependencies = new LinkedList<>();
List<Repository> repositories = new LinkedList<>();
List<File> files = new LinkedList<File>();
List<File> filesDist = new LinkedList<File>();
List<File> files = new LinkedList<>();
List<File> filesDist = new LinkedList<>();
private RepositorySystem system = Booter.newRepositorySystem();
private RepositorySystemSession session;
private RemoteRepository mavenCentral = Booter.newCentralRepository();
private RemoteRepository mavenLocal = Booter.newLocalRepository();
private List<RemoteRepository> additionalRepos = new LinkedList<RemoteRepository>();
private List<RemoteRepository> additionalRepos = new LinkedList<>();
public SparkDependencyContext(String localRepoPath, String additionalRemoteRepository) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
@ -88,11 +88,11 @@ public class SparkDependencyContext {
public void reset() {
Console.println("DepInterpreter(%dep) deprecated. "
+ "Remove dependencies and repositories through GUI interpreter menu instead.");
dependencies = new LinkedList<Dependency>();
repositories = new LinkedList<Repository>();
dependencies = new LinkedList<>();
repositories = new LinkedList<>();
files = new LinkedList<File>();
filesDist = new LinkedList<File>();
files = new LinkedList<>();
filesDist = new LinkedList<>();
}
private void addRepoFromProperty(String listOfRepo) {

16
spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java
View file

@ -114,7 +114,7 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
}
// NOTE: Must use reflection until this is exposed/fixed upstream in Scala
List<String> classPaths = new LinkedList<String>();
List<String> classPaths = new LinkedList<>();
for (URL url : urls) {
classPaths.add(url.getPath());
}
@ -151,7 +151,7 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
private MergedClassPath<AbstractFile> mergeUrlsIntoClassPath(JavaPlatform platform, URL[] urls) {
IndexedSeq<ClassPath<AbstractFile>> entries =
((MergedClassPath<AbstractFile>) platform.classPath()).entries();
List<ClassPath<AbstractFile>> cp = new LinkedList<ClassPath<AbstractFile>>();
List<ClassPath<AbstractFile>> cp = new LinkedList<>();
for (int i = 0; i < entries.size(); i++) {
cp.add(entries.apply(i));
@ -200,7 +200,7 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
return loadFromMvn(artifact, excludes, addSparkContext);
} else {
loadFromFs(artifact, addSparkContext);
LinkedList<String> libs = new LinkedList<String>();
LinkedList<String> libs = new LinkedList<>();
libs.add(artifact);
return libs;
}
@ -224,8 +224,8 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
private List<String> loadFromMvn(String artifact, Collection<String> excludes,
boolean addSparkContext) throws Exception {
List<String> loadedLibs = new LinkedList<String>();
Collection<String> allExclusions = new LinkedList<String>();
List<String> loadedLibs = new LinkedList<>();
Collection<String> allExclusions = new LinkedList<>();
allExclusions.addAll(excludes);
allExclusions.addAll(Arrays.asList(exclusions));
@ -244,8 +244,8 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
}
}
List<URL> newClassPathList = new LinkedList<URL>();
List<File> files = new LinkedList<File>();
List<URL> newClassPathList = new LinkedList<>();
List<File> files = new LinkedList<>();
for (ArtifactResult artifactResult : listOfArtifact) {
logger.info("Load " + artifactResult.getArtifact().getGroupId() + ":"
+ artifactResult.getArtifact().getArtifactId() + ":"
@ -302,7 +302,7 @@ public class SparkDependencyResolver extends AbstractDependencyResolver {
}
public static Collection<String> inferScalaVersion(Collection<String> artifact) {
List<String> list = new LinkedList<String>();
List<String> list = new LinkedList<>();
for (String a : artifact) {
list.add(inferScalaVersion(a));
}

147
spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java Normal file
View file

@ -0,0 +1,147 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.spark;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOutputListener;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import static org.junit.Assert.*;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class PySparkInterpreterTest {
public static SparkInterpreter sparkInterpreter;
public static PySparkInterpreter pySparkInterpreter;
public static InterpreterGroup intpGroup;
private File tmpDir;
public static Logger LOGGER = LoggerFactory.getLogger(PySparkInterpreterTest.class);
private InterpreterContext context;
public static Properties getPySparkTestProperties() {
Properties p = new Properties();
p.setProperty("master", "local[*]");
p.setProperty("spark.app.name", "Zeppelin Test");
p.setProperty("zeppelin.spark.useHiveContext", "true");
p.setProperty("zeppelin.spark.maxResult", "1000");
p.setProperty("zeppelin.spark.importImplicit", "true");
p.setProperty("zeppelin.pyspark.python", "python");
return p;
}
/**
* Get spark version number as a numerical value.
* eg. 1.1.x => 11, 1.2.x => 12, 1.3.x => 13 ...
*/
public static int getSparkVersionNumber() {
if (sparkInterpreter == null) {
return 0;
}
String[] split = sparkInterpreter.getSparkContext().version().split("\\.");
int version = Integer.parseInt(split[0]) * 10 + Integer.parseInt(split[1]);
return version;
}
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
tmpDir.mkdirs();
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
if (sparkInterpreter == null) {
sparkInterpreter = new SparkInterpreter(getPySparkTestProperties());
intpGroup.get("note").add(sparkInterpreter);
sparkInterpreter.setInterpreterGroup(intpGroup);
sparkInterpreter.open();
}
if (pySparkInterpreter == null) {
pySparkInterpreter = new PySparkInterpreter(getPySparkTestProperties());
intpGroup.get("note").add(pySparkInterpreter);
pySparkInterpreter.setInterpreterGroup(intpGroup);
pySparkInterpreter.open();
}
context = new InterpreterContext("note", "id", "title", "text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("id"),
new LinkedList<InterpreterContextRunner>(),
new InterpreterOutput(new InterpreterOutputListener() {
@Override
public void onAppend(InterpreterOutput out, byte[] line) {
}
@Override
public void onUpdate(InterpreterOutput out, byte[] output) {
}
}));
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
}
private void delete(File file) {
if (file.isFile()) file.delete();
else if (file.isDirectory()) {
File[] files = file.listFiles();
if (files != null && files.length > 0) {
for (File f : files) {
delete(f);
}
}
file.delete();
}
}
@Test
public void testBasicIntp() {
if (getSparkVersionNumber() > 11) {
assertEquals(InterpreterResult.Code.SUCCESS,
pySparkInterpreter.interpret("a = 1\n", context).code());
}
}
}

46
testing/downloadSpark.sh
View file

@ -16,28 +16,15 @@
# limitations under the License.
#
if [[ "$#" -ne 2 ]]; then
echo "usage) $0 [spark version] [hadoop version]"
echo " eg) $0 1.3.1 2.6"
exit 1
exit 0
fi
SPARK_VERSION="${1}"
HADOOP_VERSION="${2}"
echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
if [[ "$?" -eq 0 ]]; then
echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
if [[ "$?" -eq 0 ]]; then
SPARK_VER_RANGE="<=1.2"
else
SPARK_VER_RANGE="<=1.3"
fi
else
SPARK_VER_RANGE=">1.3"
fi
set -xe
MAX_DOWNLOAD_TIME_SEC=590
@ -75,30 +62,13 @@ if [[ ! -d "${SPARK_HOME}" ]]; then
ls -la .
echo "${SPARK_CACHE} does not have ${SPARK_ARCHIVE} downloading ..."
# download archive if not cached
if [[ "${SPARK_VERSION}" = "1.1.1" || "${SPARK_VERSION}" = "1.2.2" || "${SPARK_VERSION}" = "1.3.1" || "${SPARK_VERSION}" = "1.4.1" ]]; then
echo "${SPARK_VERSION} being downloaded from archives"
# spark old versions are only available only on the archives (prior to 1.5.2)
STARTTIME=`date +%s`
#timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
ENDTIME=`date +%s`
DOWNLOADTIME="$((ENDTIME-STARTTIME))"
else
echo "${SPARK_VERSION} being downloaded from mirror"
# spark 1.5.2 and up and later can be downloaded from mirror
# get download address from mirror
MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz?asjson=1")
PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
STARTTIME=`date +%s`
#timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget -q "${PREFFERED}${PATHINFO}"
download_with_retry "${PREFFERED}${PATHINFO}"
ENDTIME=`date +%s`
DOWNLOADTIME="$((ENDTIME-STARTTIME))"
fi
# download spark from archive if not cached
echo "${SPARK_VERSION} being downloaded from archives"
STARTTIME=`date +%s`
#timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
ENDTIME=`date +%s`
DOWNLOADTIME="$((ENDTIME-STARTTIME))"
fi
# extract archive in un-cached root, clean-up on failure

12
zeppelin-distribution/README.md
View file

@ -23,18 +23,12 @@ Apache Zeppelin is distributed as a single gzip archive with the following struc
Zeppelin
├── bin
│ ├── zeppelin.sh
│ └── seppelin-deamon.sh
│ └── zeppelin-daemon.sh
├── conf
├── interpreter
├── lib
├── licenses
├── zan-repo
│ ├── txt.wordcount
│ ├── vis.bubble
│ ├── vis.gchart
│ ├── ml.something
│ └── ...
├── zeppelin-server-<verion>.jar
├── notebook
└── zeppelin-web-<verion>.war
```
@ -42,4 +36,4 @@ Zeppelin
We use `maven-assembly-plugin` to build it, see `zeppelin-distribution/src/assemble/distribution.xml ` for details.
>**IMPORTANT:** `_/lib_` subdirectory contains all transitive dependencies of the `zeppelin-distribution` module,
automatically resolved by maven, except for explicitly excluded `_server_` and `_web_` Zeppelin sub-modules.
automatically resolved by maven, except for explicitly excluded `_web_` Zeppelin sub-modules.

2
zeppelin-distribution/src/assemble/distribution.xml
View file

@ -33,7 +33,6 @@
<useAllReactorProjects>true</useAllReactorProjects> -->
<!-- Now, select which projects to include in this module-set. -->
<includes>
<include>org.apache.zeppelin:zeppelin-server</include>
<include>org.apache.zeppelin:zeppelin-web</include>
</includes>
<useProjectArtifact>false</useProjectArtifact>
@ -44,7 +43,6 @@
<useProjectArtifact>false</useProjectArtifact>
<excludes>
<exclude>${project.groupId}:zeppelin-web</exclude>
<exclude>${project.groupId}:zeppelin-server</exclude>
</excludes>
</dependencySet>
</dependencySets>

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java
View file

@ -37,7 +37,7 @@ import org.sonatype.aether.resolution.ArtifactResult;
*/
public abstract class AbstractDependencyResolver {
protected RepositorySystem system = Booter.newRepositorySystem();
protected List<RemoteRepository> repos = new LinkedList<RemoteRepository>();
protected List<RemoteRepository> repos = new LinkedList<>();
protected RepositorySystemSession session;
public AbstractDependencyResolver(String localRepoPath) {

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Dependency.java
View file

@ -31,7 +31,7 @@ public class Dependency {
public Dependency(String groupArtifactVersion) {
this.groupArtifactVersion = groupArtifactVersion;
exclusions = new LinkedList<String>();
exclusions = new LinkedList<>();
}
@Override

16
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java
View file

@ -42,11 +42,11 @@ import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
*
*/
public class DependencyContext {
List<Dependency> dependencies = new LinkedList<Dependency>();
List<Repository> repositories = new LinkedList<Repository>();
List<Dependency> dependencies = new LinkedList<>();
List<Repository> repositories = new LinkedList<>();
List<File> files = new LinkedList<File>();
List<File> filesDist = new LinkedList<File>();
List<File> files = new LinkedList<>();
List<File> filesDist = new LinkedList<>();
private RepositorySystem system = Booter.newRepositorySystem();
private RepositorySystemSession session;
private RemoteRepository mavenCentral = Booter.newCentralRepository();
@ -73,11 +73,11 @@ public class DependencyContext {
}
public void reset() {
dependencies = new LinkedList<Dependency>();
repositories = new LinkedList<Repository>();
dependencies = new LinkedList<>();
repositories = new LinkedList<>();
files = new LinkedList<File>();
filesDist = new LinkedList<File>();
files = new LinkedList<>();
filesDist = new LinkedList<>();
}

10
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
View file

@ -70,7 +70,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
throws RepositoryException, IOException {
if (StringUtils.isBlank(artifact)) {
// Skip dependency loading if artifact is empty
return new LinkedList<File>();
return new LinkedList<>();
}
// <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>
@ -78,7 +78,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
if (numSplits >= 3 && numSplits <= 6) {
return loadFromMvn(artifact, excludes);
} else {
LinkedList<File> libs = new LinkedList<File>();
LinkedList<File> libs = new LinkedList<>();
libs.add(new File(artifact));
return libs;
}
@ -90,7 +90,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
public List<File> load(String artifact, Collection<String> excludes, File destPath)
throws RepositoryException, IOException {
List<File> libs = new LinkedList<File>();
List<File> libs = new LinkedList<>();
if (StringUtils.isNotBlank(artifact)) {
libs = load(artifact, excludes);
@ -123,7 +123,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
private List<File> loadFromMvn(String artifact, Collection<String> excludes)
throws RepositoryException {
Collection<String> allExclusions = new LinkedList<String>();
Collection<String> allExclusions = new LinkedList<>();
allExclusions.addAll(excludes);
allExclusions.addAll(Arrays.asList(exclusions));
@ -142,7 +142,7 @@ public class DependencyResolver extends AbstractDependencyResolver {
}
}
List<File> files = new LinkedList<File>();
List<File> files = new LinkedList<>();
for (ArtifactResult artifactResult : listOfArtifact) {
files.add(artifactResult.getArtifact().getFile());
logger.info("load {}", artifactResult.getArtifact().getFile().getAbsolutePath());

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java
View file

@ -37,7 +37,7 @@ public class TransferListener extends AbstractTransferListener {
Logger logger = LoggerFactory.getLogger(TransferListener.class);
private PrintStream out;
private Map<TransferResource, Long> downloads = new ConcurrentHashMap<TransferResource, Long>();
private Map<TransferResource, Long> downloads = new ConcurrentHashMap<>();
private int lastLength;

5
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObject.java
View file

@ -38,8 +38,7 @@ public class AngularObject<T> {
private T object;
private transient AngularObjectListener listener;
private transient List<AngularObjectWatcher> watchers
= new LinkedList<AngularObjectWatcher>();
private transient List<AngularObjectWatcher> watchers = new LinkedList<>();
private String noteId; // noteId belonging to. null for global scope
private String paragraphId; // paragraphId belongs to. null for notebook scope
@ -175,7 +174,7 @@ public class AngularObject<T> {
}
final Logger logger = LoggerFactory.getLogger(AngularObject.class);
List<AngularObjectWatcher> ws = new LinkedList<AngularObjectWatcher>();
List<AngularObjectWatcher> ws = new LinkedList<>();
synchronized (watchers) {
ws.addAll(watchers);
}

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java
View file

@ -32,8 +32,7 @@ import java.util.Map;
* - Global scope : Shared to all notebook that uses the same interpreter group
*/
public class AngularObjectRegistry {
Map<String, Map<String, AngularObject>> registry =
new HashMap<String, Map<String, AngularObject>>();
Map<String, Map<String, AngularObject>> registry = new HashMap<>();
private final String GLOBAL_KEY = "_GLOBAL_";
private AngularObjectRegistryListener listener;
private String interpreterId;
@ -209,7 +208,7 @@ public class AngularObjectRegistry {
* @return all angularobject in the scope
*/
public List<AngularObject> getAll(String noteId, String paragraphId) {
List<AngularObject> all = new LinkedList<AngularObject>();
List<AngularObject> all = new LinkedList<>();
synchronized (registry) {
Map<String, AngularObject> r = getRegistryForKey(noteId, paragraphId);
if (r != null) {
@ -228,7 +227,7 @@ public class AngularObjectRegistry {
* @return
*/
public List<AngularObject> getAllWithGlobal(String noteId) {
List<AngularObject> all = new LinkedList<AngularObject>();
List<AngularObject> all = new LinkedList<>();
synchronized (registry) {
Map<String, AngularObject> global = getRegistryForKey(null, null);
if (global != null) {

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/GUI.java
View file

@ -32,8 +32,8 @@ import org.apache.zeppelin.display.Input.ParamOption;
*/
public class GUI implements Serializable {
Map<String, Object> params = new HashMap<String, Object>(); // form parameters from client
Map<String, Input> forms = new TreeMap<String, Input>(); // form configuration
Map<String, Object> params = new HashMap<>(); // form parameters from client
Map<String, Input> forms = new TreeMap<>(); // form configuration
public GUI() {
@ -86,7 +86,7 @@ public class GUI implements Serializable {
checked = defaultChecked;
}
forms.put(id, new Input(id, defaultChecked, "checkbox", options));
Collection<Object> filtered = new LinkedList<Object>();
Collection<Object> filtered = new LinkedList<>();
for (Object o : checked) {
if (isValidOption(o, options)) {
filtered.add(o);
@ -105,6 +105,6 @@ public class GUI implements Serializable {
}
public void clear() {
this.forms = new TreeMap<String, Input>();
this.forms = new TreeMap<>();
}
}

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/Input.java
View file

@ -292,7 +292,7 @@ public class Input implements Serializable {
}
public static Map<String, Input> extractSimpleQueryParam(String script) {
Map<String, Input> params = new HashMap<String, Input>();
Map<String, Input> params = new HashMap<>();
if (script == null) {
return params;
}
@ -331,7 +331,7 @@ public class Input implements Serializable {
}
Collection<Object> checked = value instanceof Collection ? (Collection<Object>) value
: Arrays.asList((Object[]) value);
List<Object> validChecked = new LinkedList<Object>();
List<Object> validChecked = new LinkedList<>();
for (Object o : checked) { // filter out obsolete checked values
for (ParamOption option : input.getOptions()) {
if (option.getValue().equals(o)) {
@ -387,14 +387,14 @@ public class Input implements Serializable {
public static String[] split(String str, String escapeSeq, char escapeChar, String[] blockStart,
String[] blockEnd, String[] splitters, boolean includeSplitter) {
List<String> splits = new ArrayList<String>();
List<String> splits = new ArrayList<>();
String curString = "";
boolean escape = false; // true when escape char is found
int lastEscapeOffset = -1;
int blockStartPos = -1;
List<Integer> blockStack = new LinkedList<Integer>();
List<Integer> blockStack = new LinkedList<>();
for (int i = 0; i < str.length(); i++) {
char c = str.charAt(i);

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java
View file

@ -200,7 +200,7 @@ public class ApplicationLoader {
}
// Create Application classloader
List<URL> urlList = new LinkedList<URL>();
List<URL> urlList = new LinkedList<>();
// load artifact
if (packageInfo.getArtifact() != null) {

3
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterContext.java
View file

@ -30,8 +30,7 @@ import org.apache.zeppelin.resource.ResourcePool;
* Interpreter context
*/
public class InterpreterContext {
private static final ThreadLocal<InterpreterContext> threadIC =
new ThreadLocal<InterpreterContext>();
private static final ThreadLocal<InterpreterContext> threadIC = new ThreadLocal<>();
public final InterpreterOutput out;

10
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -55,7 +55,7 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
// List<Interpreter>>();
private static final Map<String, InterpreterGroup> allInterpreterGroups =
new ConcurrentHashMap<String, InterpreterGroup>();
new ConcurrentHashMap<>();
public static InterpreterGroup getByInterpreterGroupId(String id) {
return allInterpreterGroups.get(id);
@ -147,7 +147,7 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
*/
public void close() {
LOGGER.info("Close interpreter group " + getId());
List<Interpreter> intpToClose = new LinkedList<Interpreter>();
List<Interpreter> intpToClose = new LinkedList<>();
for (List<Interpreter> intpGroupForNote : this.values()) {
intpToClose.addAll(intpGroupForNote);
}
@ -168,7 +168,7 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
if (intpToClose == null) {
return;
}
List<Thread> closeThreads = new LinkedList<Thread>();
List<Thread> closeThreads = new LinkedList<>();
for (final Interpreter intp : intpToClose) {
Thread t = new Thread() {
@ -219,7 +219,7 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
*/
public void destroy() {
LOGGER.info("Destroy interpreter group " + getId());
List<Interpreter> intpToDestroy = new LinkedList<Interpreter>();
List<Interpreter> intpToDestroy = new LinkedList<>();
for (List<Interpreter> intpGroupForNote : this.values()) {
intpToDestroy.addAll(intpGroupForNote);
}
@ -241,7 +241,7 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
return;
}
List<Thread> destroyThreads = new LinkedList<Thread>();
List<Thread> destroyThreads = new LinkedList<>();
for (final Interpreter intp : intpToDestroy) {
Thread t = new Thread() {

3
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterHookRegistry.java
View file

@ -29,8 +29,7 @@ import java.util.Map;
public class InterpreterHookRegistry {
public static final String GLOBAL_KEY = "_GLOBAL_";
private String interpreterId;
private Map<String, Map<String, Map<String, String>>> registry =
new HashMap<String, Map<String, Map<String, String>>>();
private Map<String, Map<String, Map<String, String>>> registry = new HashMap<>();
/**
* hookRegistry constructor.

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java
View file

@ -39,7 +39,7 @@ public class InterpreterOutput extends OutputStream {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
private final List<Object> outList = new LinkedList<Object>();
private final List<Object> outList = new LinkedList<>();
private InterpreterOutputChangeWatcher watcher;
private final InterpreterOutputListener flushListener;
private InterpreterResult.Type type = InterpreterResult.Type.TEXT;
@ -185,7 +185,7 @@ public class InterpreterOutput extends OutputStream {
public byte[] toByteArray() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
List<Object> all = new LinkedList<Object>();
List<Object> all = new LinkedList<>();
synchronized (outList) {
all.addAll(outList);

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcher.java
View file

@ -44,8 +44,8 @@ public class InterpreterOutputChangeWatcher extends Thread {
Logger logger = LoggerFactory.getLogger(InterpreterOutputChangeWatcher.class);
private WatchService watcher;
private final List<File> watchFiles = new LinkedList<File>();
private final Map<WatchKey, File> watchKeys = new HashMap<WatchKey, File>();
private final List<File> watchFiles = new LinkedList<>();
private final Map<WatchKey, File> watchKeys = new HashMap<>();
private InterpreterOutputChangeListener listener;
private boolean stop;

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterPropertyBuilder.java
View file

@ -24,7 +24,7 @@ import java.util.Map;
* InterpreterPropertyBuilder
*/
public class InterpreterPropertyBuilder {
Map<String, InterpreterProperty> properties = new HashMap<String, InterpreterProperty>();
Map<String, InterpreterProperty> properties = new HashMap<>();
public InterpreterPropertyBuilder add(String name, String defaultValue, String description){
properties.put(name, new InterpreterProperty(defaultValue, description));

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java
View file

@ -119,7 +119,7 @@ public class InterpreterResult implements Serializable {
private TreeMap<Integer, Type> buildIndexMap(String msg) {
int lastIndexOftypes = 0;
TreeMap<Integer, Type> typesLastIndexInMsg = new TreeMap<Integer, Type>();
TreeMap<Integer, Type> typesLastIndexInMsg = new TreeMap<>();
Type[] types = Type.values();
for (Type t : types) {
lastIndexOftypes = getIndexOfType(msg, t);

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java
View file

@ -105,7 +105,7 @@ public class DevInterpreter extends Interpreter {
@Override
public List<InterpreterCompletion> completion(String buf, int cursor) {
return new LinkedList<InterpreterCompletion>();
return new LinkedList<>();
}
public InterpreterContext getLastInterpretContext() {

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
View file

@ -37,7 +37,7 @@ import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
public class ClientFactory extends BasePooledObjectFactory<Client>{
private String host;
private int port;
Map<Client, TSocket> clientSocketMap = new HashMap<Client, TSocket>();
Map<Client, TSocket> clientSocketMap = new HashMap<>();
public ClientFactory(String host, int port) {
this.host = host;
@ -64,7 +64,7 @@ public class ClientFactory extends BasePooledObjectFactory<Client>{
@Override
public PooledObject<Client> wrap(Client client) {
return new DefaultPooledObject<Client>(client);
return new DefaultPooledObject<>(client);
}
@Override

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java
View file

@ -35,7 +35,7 @@ public class InterpreterContextRunnerPool {
private Map<String, List<InterpreterContextRunner>> interpreterContextRunners;
public InterpreterContextRunnerPool() {
interpreterContextRunners = new HashMap<String, List<InterpreterContextRunner>>();
interpreterContextRunners = new HashMap<>();
}

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -141,7 +141,7 @@ public class RemoteInterpreter extends Interpreter {
}
private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
Map<String, String> env = new HashMap<String, String>();
Map<String, String> env = new HashMap<>();
for (Object key : property.keySet()) {
if (isEnvString((String) key)) {
env.put((String) key, property.getProperty((String) key));

18
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java
View file

@ -43,9 +43,9 @@ import java.util.Map;
*/
public class RemoteInterpreterEventClient implements ResourcePoolConnector {
private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEvent.class);
private final List<RemoteInterpreterEvent> eventQueue = new LinkedList<RemoteInterpreterEvent>();
private final List<ResourceSet> getAllResourceResponse = new LinkedList<ResourceSet>();
private final Map<ResourceId, Object> getResourceResponse = new HashMap<ResourceId, Object>();
private final List<RemoteInterpreterEvent> eventQueue = new LinkedList<>();
private final List<ResourceSet> getAllResourceResponse = new LinkedList<>();
private final Map<ResourceId, Object> getResourceResponse = new HashMap<>();
private final Gson gson = new Gson();
/**
@ -79,7 +79,7 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
* notify angularObject removal
*/
public void angularObjectRemove(String name, String noteId, String paragraphId) {
Map<String, String> removeObject = new HashMap<String, String>();
Map<String, String> removeObject = new HashMap<>();
removeObject.put("name", name);
removeObject.put("noteId", noteId);
removeObject.put("paragraphId", paragraphId);
@ -213,7 +213,7 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
}
public void onInterpreterOutputAppend(String noteId, String paragraphId, String output) {
Map<String, String> appendOutput = new HashMap<String, String>();
Map<String, String> appendOutput = new HashMap<>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("data", output);
@ -224,7 +224,7 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
}
public void onInterpreterOutputUpdate(String noteId, String paragraphId, String output) {
Map<String, String> appendOutput = new HashMap<String, String>();
Map<String, String> appendOutput = new HashMap<>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("data", output);
@ -243,7 +243,7 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
}
public void onAppOutputAppend(String noteId, String paragraphId, String appId, String output) {
Map<String, String> appendOutput = new HashMap<String, String>();
Map<String, String> appendOutput = new HashMap<>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("appId", appId);
@ -256,7 +256,7 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
public void onAppOutputUpdate(String noteId, String paragraphId, String appId, String output) {
Map<String, String> appendOutput = new HashMap<String, String>();
Map<String, String> appendOutput = new HashMap<>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("appId", appId);
@ -268,7 +268,7 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector {
}
public void onAppStatusUpdate(String noteId, String paragraphId, String appId, String status) {
Map<String, String> appendOutput = new HashMap<String, String>();
Map<String, String> appendOutput = new HashMap<>();
appendOutput.put("noteId", noteId);
appendOutput.put("paragraphId", paragraphId);
appendOutput.put("appId", appId);

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -211,7 +211,7 @@ public class RemoteInterpreterEventPoller extends Thread {
boolean broken = false;
try {
client = interpreterProcess.getClient();
List<String> resourceList = new LinkedList<String>();
List<String> resourceList = new LinkedList<>();
Gson gson = new Gson();
for (Resource r : resourceSet) {
resourceList.add(gson.toJson(r));

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
View file

@ -78,7 +78,7 @@ public abstract class RemoteInterpreterProcess {
}
if (clientPool == null) {
clientPool = new GenericObjectPool<Client>(new ClientFactory(getHost(), getPort()));
clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
clientPool.setTestOnBorrow(true);
remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup);

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -82,7 +82,7 @@ public class RemoteInterpreterServer
public RemoteInterpreterServer(int port) throws TTransportException {
this.port = port;
processor = new RemoteInterpreterService.Processor<RemoteInterpreterServer>(this);
processor = new RemoteInterpreterService.Processor<>(this);
TServerSocket serverTransport = new TServerSocket(port);
server = new TThreadPoolServer(
new TThreadPoolServer.Args(serverTransport).processor(processor));
@ -181,7 +181,7 @@ public class RemoteInterpreterServer
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(noteId);
if (interpreters == null) {
interpreters = new LinkedList<Interpreter>();
interpreters = new LinkedList<>();
interpreterGroup.put(noteId, interpreters);
}
@ -532,7 +532,7 @@ public class RemoteInterpreterServer
}
private InterpreterContext convert(RemoteInterpreterContext ric, InterpreterOutput output) {
List<InterpreterContextRunner> contextRunners = new LinkedList<InterpreterContextRunner>();
List<InterpreterContextRunner> contextRunners = new LinkedList<>();
List<InterpreterContextRunner> runners = gson.fromJson(ric.getRunners(),
new TypeToken<List<RemoteInterpreterContextRunner>>() {
}.getType());
@ -775,7 +775,7 @@ public class RemoteInterpreterServer
@Override
public List<String> resourcePoolGetAll() throws TException {
logger.debug("Request getAll from ZeppelinServer");
List<String> result = new LinkedList<String>();
List<String> result = new LinkedList<>();
if (resourcePool == null) {
return result;

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ExecutorFactory.java
View file

@ -28,7 +28,7 @@ public class ExecutorFactory {
private static ExecutorFactory _executor;
private static Long _executorLock = new Long(0);
Map<String, ExecutorService> executor = new HashMap<String, ExecutorService>();
Map<String, ExecutorService> executor = new HashMap<>();
public ExecutorFactory() {

6
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/FIFOScheduler.java
View file

@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory;
* FIFOScheduler runs submitted job sequentially
*/
public class FIFOScheduler implements Scheduler {
List<Job> queue = new LinkedList<Job>();
List<Job> queue = new LinkedList<>();
private ExecutorService executor;
private SchedulerListener listener;
boolean terminate = false;
@ -53,7 +53,7 @@ public class FIFOScheduler implements Scheduler {
@Override
public Collection<Job> getJobsWaiting() {
List<Job> ret = new LinkedList<Job>();
List<Job> ret = new LinkedList<>();
synchronized (queue) {
for (Job job : queue) {
ret.add(job);
@ -64,7 +64,7 @@ public class FIFOScheduler implements Scheduler {
@Override
public Collection<Job> getJobsRunning() {
List<Job> ret = new LinkedList<Job>();
List<Job> ret = new LinkedList<>();
Job job = runningJob;
if (job != null) {

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/ParallelScheduler.java
View file

@ -31,8 +31,8 @@ import org.slf4j.LoggerFactory;
* Parallel scheduler runs submitted job concurrently.
*/
public class ParallelScheduler implements Scheduler {
List<Job> queue = new LinkedList<Job>();
List<Job> running = new LinkedList<Job>();
List<Job> queue = new LinkedList<>();
List<Job> running = new LinkedList<>();
private ExecutorService executor;
private SchedulerListener listener;
boolean terminate = false;
@ -56,7 +56,7 @@ public class ParallelScheduler implements Scheduler {
@Override
public Collection<Job> getJobsWaiting() {
List<Job> ret = new LinkedList<Job>();
List<Job> ret = new LinkedList<>();
synchronized (queue) {
for (Job job : queue) {
ret.add(job);
@ -82,7 +82,7 @@ public class ParallelScheduler implements Scheduler {
@Override
public Collection<Job> getJobsRunning() {
List<Job> ret = new LinkedList<Job>();
List<Job> ret = new LinkedList<>();
synchronized (queue) {
for (Job job : running) {
ret.add(job);

8
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
View file

@ -39,8 +39,8 @@ import java.util.concurrent.ExecutorService;
public class RemoteScheduler implements Scheduler {
Logger logger = LoggerFactory.getLogger(RemoteScheduler.class);
List<Job> queue = new LinkedList<Job>();
List<Job> running = new LinkedList<Job>();
List<Job> queue = new LinkedList<>();
List<Job> running = new LinkedList<>();
private ExecutorService executor;
private SchedulerListener listener;
boolean terminate = false;
@ -105,7 +105,7 @@ public class RemoteScheduler implements Scheduler {
@Override
public Collection<Job> getJobsWaiting() {
List<Job> ret = new LinkedList<Job>();
List<Job> ret = new LinkedList<>();
synchronized (queue) {
for (Job job : queue) {
ret.add(job);
@ -131,7 +131,7 @@ public class RemoteScheduler implements Scheduler {
@Override
public Collection<Job> getJobsRunning() {
List<Job> ret = new LinkedList<Job>();
List<Job> ret = new LinkedList<>();
synchronized (queue) {
for (Job job : running) {
ret.add(job);

4
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
View file

@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
public class SchedulerFactory implements SchedulerListener {
private static final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
ExecutorService executor;
Map<String, Scheduler> schedulers = new LinkedHashMap<String, Scheduler>();
Map<String, Scheduler> schedulers = new LinkedHashMap<>();
private static SchedulerFactory singleton;
private static Long singletonLock = new Long(0);
@ -117,7 +117,7 @@ public class SchedulerFactory implements SchedulerListener {
}
public Collection<Scheduler> listScheduler(String name) {
List<Scheduler> s = new LinkedList<Scheduler>();
List<Scheduler> s = new LinkedList<>();
synchronized (schedulers) {
for (Scheduler ss : schedulers.values()) {
s.add(ss);

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/display/InputTest.java
View file

@ -102,7 +102,7 @@ public class InputTest {
// test form substitution without new forms
String script = "INPUT=${input_form=}SELECTED=${select_form(Selection Form)=,s_op1|s_op2|s_op3}\n" +
"CHECKED=${checkbox:checkbox_form=c_op1|c_op2,c_op1|c_op2|c_op3}";
Map<String, Object> params = new HashMap<String, Object>();
Map<String, Object> params = new HashMap<>();
params.put("input_form", "some_input");
params.put("select_form", "s_op2");
params.put("checkbox_form", new String[]{"c_op1", "c_op3"});

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
View file

@ -59,7 +59,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
intpGroup = new InterpreterGroup("intpId");
localRegistry = new RemoteAngularObjectRegistry("intpId", this, intpGroup);
intpGroup.setAngularObjectRegistry(localRegistry);
env = new HashMap<String, String>();
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
Properties p = new Properties();

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
View file

@ -50,7 +50,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
env = new HashMap<String, String>();
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
}

6
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
View file

@ -65,7 +65,7 @@ public class RemoteInterpreterTest {
@Before
public void setUp() throws Exception {
intpGroup = new InterpreterGroup();
env = new HashMap<String, String>();
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
}
@ -390,7 +390,7 @@ public class RemoteInterpreterTest {
intpA.open();
int concurrency = 3;
final List<String> results = new LinkedList<String>();
final List<String> results = new LinkedList<>();
Scheduler scheduler = intpA.getScheduler();
for (int i = 0; i < concurrency; i++) {
@ -469,7 +469,7 @@ public class RemoteInterpreterTest {
int concurrency = 4;
final int timeToSleep = 1000;
final List<String> results = new LinkedList<String>();
final List<String> results = new LinkedList<>();
long start = System.currentTimeMillis();
Scheduler scheduler = intpA.getScheduler();

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
View file

@ -55,7 +55,7 @@ public class DistributedResourcePoolTest {
@Before
public void setUp() throws Exception {
env = new HashMap<String, String>();
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
Properties p = new Properties();

4
zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
View file

@ -68,7 +68,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
public void test() throws Exception {
Properties p = new Properties();
final InterpreterGroup intpGroup = new InterpreterGroup();
Map<String, String> env = new HashMap<String, String>();
Map<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(
@ -158,7 +158,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
public void testAbortOnPending() throws Exception {
Properties p = new Properties();
final InterpreterGroup intpGroup = new InterpreterGroup();
Map<String, String> env = new HashMap<String, String>();
Map<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(

2
zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/SleepingJob.java
View file

@ -71,7 +71,7 @@ public class SleepingJob extends Job{
@Override
public Map<String, Object> info() {
Map<String, Object> i = new HashMap<String, Object>();
Map<String, Object> i = new HashMap<>();
i.put("LoopCount", Integer.toString(count));
return i;
}

2
zeppelin-server/pom.xml
View file

@ -33,7 +33,7 @@
<name>Zeppelin: Server</name>
<properties>
<cxf.version>2.7.7</cxf.version>
<cxf.version>2.7.8</cxf.version>
<commons.httpclient.version>4.3.6</commons.httpclient.version>
<hadoop-common.version>2.6.0</hadoop-common.version>
</properties>

12
zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
View file

@ -33,6 +33,7 @@ import javax.ws.rs.core.Response.Status;
import com.google.gson.Gson;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.zeppelin.rest.message.RestartInterpreterRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.repository.RemoteRepository;
@ -150,10 +151,15 @@ public class InterpreterRestApi {
@PUT
@Path("setting/restart/{settingId}")
@ZeppelinApi
public Response restartSetting(@PathParam("settingId") String settingId) {
logger.info("Restart interpreterSetting {}", settingId);
public Response restartSetting(String message, @PathParam("settingId") String settingId) {
logger.info("Restart interpreterSetting {}, msg={}", settingId, message);
try {
interpreterFactory.restart(settingId);
RestartInterpreterRequest request = gson.fromJson(message, RestartInterpreterRequest.class);
String noteId = request == null ? null : request.getNoteId();
interpreterFactory.restart(settingId, noteId);
} catch (InterpreterException e) {
logger.error("Exception in InterpreterRestApi while restartSetting ", e);
return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e))

238
zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
View file

@ -23,6 +23,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@ -34,31 +35,33 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import com.google.common.collect.Sets;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.utils.InterpreterBindingUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.zeppelin.annotation.ZeppelinApi;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.rest.exception.NotFoundException;
import org.apache.zeppelin.rest.exception.ForbiddenException;
import org.apache.zeppelin.rest.message.CronRequest;
import org.apache.zeppelin.types.InterpreterSettingsList;
import org.apache.zeppelin.rest.message.NewNoteRequest;
import org.apache.zeppelin.rest.message.NewParagraphRequest;
import org.apache.zeppelin.rest.message.RunParagraphWithParametersRequest;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.server.JsonResponse;
import org.apache.zeppelin.socket.NotebookServer;
import org.apache.zeppelin.types.InterpreterSettingsList;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.utils.InterpreterBindingUtils;
import org.apache.zeppelin.utils.SecurityUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
/**
* Rest api endpoint for the notebook.
@ -90,6 +93,8 @@ public class NotebookRestApi {
@Path("{noteId}/permissions")
@ZeppelinApi
public Response getNotePermissions(@PathParam("noteId") String noteId) {
checkIfUserCanRead(noteId,
"Insufficient privileges you cannot get the list of permissions for this note");
HashMap<String, Set<String>> permissionsMap = new HashMap<>();
permissionsMap.put("owners", notebookAuthorization.getOwners(noteId));
permissionsMap.put("readers", notebookAuthorization.getReaders(noteId));
@ -105,6 +110,60 @@ public class NotebookRestApi {
"User belongs to: " + current.toString();
}
/**
* Set of utils method to check if current user can perform action to the note.
* Since we only have security on notebook level, from now we keep this logic in this class.
* In the future we might want to generalize this for the rest of the api enmdpoints.
*/
/**
* Check if the current user own the given note.
*/
private void checkIfUserIsOwner(String noteId, String errorMsg) {
Set<String> userAndRoles = Sets.newHashSet();
userAndRoles.add(SecurityUtils.getPrincipal());
userAndRoles.addAll(SecurityUtils.getRoles());
if (!notebookAuthorization.isOwner(userAndRoles, noteId)) {
throw new ForbiddenException(errorMsg);
}
}
/**
* Check if the current user is either Owner or Writer for the given note.
*/
private void checkIfUserCanWrite(String noteId, String errorMsg) {
Set<String> userAndRoles = Sets.newHashSet();
userAndRoles.add(SecurityUtils.getPrincipal());
userAndRoles.addAll(SecurityUtils.getRoles());
if (!notebookAuthorization.hasWriteAuthorization(userAndRoles, noteId)) {
throw new ForbiddenException(errorMsg);
}
}
/**
* Check if the current user can access (at least he have to be reader) the given note.
*/
private void checkIfUserCanRead(String noteId, String errorMsg) {
Set<String> userAndRoles = Sets.newHashSet();
userAndRoles.add(SecurityUtils.getPrincipal());
userAndRoles.addAll(SecurityUtils.getRoles());
if (!notebookAuthorization.hasReadAuthorization(userAndRoles, noteId)) {
throw new ForbiddenException(errorMsg);
}
}
private void checkIfNoteIsNotNull(Note note) {
if (note == null) {
throw new NotFoundException("note not found");
}
}
private void checkIfParagraphIsNotNull(Paragraph paragraph) {
if (paragraph == null) {
throw new NotFoundException("paragraph not found");
}
}
/**
* set note authorization information
*/
@ -113,22 +172,21 @@ public class NotebookRestApi {
@ZeppelinApi
public Response putNotePermissions(@PathParam("noteId") String noteId, String req)
throws IOException {
HashMap<String, HashSet<String>> permMap =
gson.fromJson(req, new TypeToken<HashMap<String, HashSet<String>>>() {
}.getType());
Note note = notebook.getNote(noteId);
String principal = SecurityUtils.getPrincipal();
HashSet<String> roles = SecurityUtils.getRoles();
LOG.info("Set permissions {} {} {} {} {}", noteId, principal, permMap.get("owners"),
permMap.get("readers"), permMap.get("writers"));
HashSet<String> userAndRoles = new HashSet<>();
userAndRoles.add(principal);
userAndRoles.addAll(roles);
if (!notebookAuthorization.isOwner(noteId, userAndRoles)) {
return new JsonResponse<>(Status.FORBIDDEN,
ownerPermissionError(userAndRoles, notebookAuthorization.getOwners(noteId))).build();
}
checkIfUserIsOwner(noteId,
ownerPermissionError(userAndRoles, notebookAuthorization.getOwners(noteId)));
HashMap<String, HashSet<String>> permMap =
gson.fromJson(req, new TypeToken<HashMap<String, HashSet<String>>>() {}.getType());
Note note = notebook.getNote(noteId);
LOG.info("Set permissions {} {} {} {} {}", noteId, principal, permMap.get("owners"),
permMap.get("readers"), permMap.get("writers"));
HashSet<String> readers = permMap.get("readers");
HashSet<String> owners = permMap.get("owners");
@ -170,6 +228,9 @@ public class NotebookRestApi {
@Path("interpreter/bind/{noteId}")
@ZeppelinApi
public Response bind(@PathParam("noteId") String noteId, String req) throws IOException {
checkIfUserCanWrite(noteId,
"Insufficient privileges you cannot bind any interpreters to this note");
List<String> settingIdList = gson.fromJson(req, new TypeToken<List<String>>() {
}.getType());
notebook.bindInterpretersToNote(SecurityUtils.getPrincipal(), noteId, settingIdList);
@ -183,6 +244,8 @@ public class NotebookRestApi {
@Path("interpreter/bind/{noteId}")
@ZeppelinApi
public Response bind(@PathParam("noteId") String noteId) {
checkIfUserCanRead(noteId, "Insufficient privileges you cannot get any interpreters settings");
List<InterpreterSettingsList> settingList =
InterpreterBindingUtils.getInterpreterBindings(notebook, noteId);
notebookServer.broadcastInterpreterBindings(noteId, settingList);
@ -204,9 +267,8 @@ public class NotebookRestApi {
@ZeppelinApi
public Response getNote(@PathParam("noteId") String noteId) throws IOException {
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanRead(noteId, "Insufficient privileges you cannot get this note");
return new JsonResponse<>(Status.OK, "", note).build();
}
@ -222,6 +284,7 @@ public class NotebookRestApi {
@Path("export/{noteId}")
@ZeppelinApi
public Response exportNote(@PathParam("noteId") String noteId) throws IOException {
checkIfUserCanRead(noteId, "Insufficient privileges you cannot export this note");
String exportJson = notebook.exportNote(noteId);
return new JsonResponse<>(Status.OK, "", exportJson).build();
}
@ -290,6 +353,7 @@ public class NotebookRestApi {
@ZeppelinApi
public Response deleteNote(@PathParam("noteId") String noteId) throws IOException {
LOG.info("Delete note {} ", noteId);
checkIfUserIsOwner(noteId, "Insufficient privileges you cannot delete this note");
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
if (!(noteId.isEmpty())) {
Note note = notebook.getNote(noteId);
@ -315,6 +379,7 @@ public class NotebookRestApi {
public Response cloneNote(@PathParam("noteId") String noteId, String message)
throws IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("clone note by JSON {}", message);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot clone this note");
NewNoteRequest request = gson.fromJson(message, NewNoteRequest.class);
String newNoteName = null;
if (request != null) {
@ -342,9 +407,8 @@ public class NotebookRestApi {
LOG.info("insert paragraph {} {}", noteId, message);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot add paragraph to this note");
NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class);
@ -379,14 +443,10 @@ public class NotebookRestApi {
LOG.info("get paragraph {} {}", noteId, paragraphId);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanRead(noteId, "Insufficient privileges you cannot get this paragraph");
Paragraph p = note.getParagraph(paragraphId);
if (p == null) {
return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build();
}
checkIfParagraphIsNotNull(p);
return new JsonResponse<>(Status.OK, "", p).build();
}
@ -407,14 +467,11 @@ public class NotebookRestApi {
LOG.info("move paragraph {} {} {}", noteId, paragraphId, newIndex);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot move paragraph");
Paragraph p = note.getParagraph(paragraphId);
if (p == null) {
return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build();
}
checkIfParagraphIsNotNull(p);
try {
note.moveParagraph(paragraphId, Integer.parseInt(newIndex), true);
@ -444,14 +501,12 @@ public class NotebookRestApi {
LOG.info("delete paragraph {} {}", noteId, paragraphId);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanRead(noteId,
"Insufficient privileges you cannot remove paragraph from this note");
Paragraph p = note.getParagraph(paragraphId);
if (p == null) {
return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build();
}
checkIfParagraphIsNotNull(p);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.removeParagraph(SecurityUtils.getPrincipal(), paragraphId);
@ -461,6 +516,27 @@ public class NotebookRestApi {
return new JsonResponse(Status.OK, "").build();
}
/**
* Clear result of all paragraphs REST API
*
* @param noteId ID of Note
* @return JSON with status.ok
*/
@PUT
@Path("{noteId}/clear")
@ZeppelinApi
public Response clearAllParagraphOutput(@PathParam("noteId") String noteId)
throws IOException {
LOG.info("clear all paragraph output of note {}", noteId);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot clear this note");
Note note = notebook.getNote(noteId);
checkIfNoteIsNotNull(note);
note.clearAllParagraphOutput();
return new JsonResponse(Status.OK, "").build();
}
/**
* Run note jobs REST API
*
@ -475,9 +551,8 @@ public class NotebookRestApi {
throws IOException, IllegalArgumentException {
LOG.info("run note jobs {} ", noteId);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot run job for this note");
try {
note.runAll();
@ -504,9 +579,8 @@ public class NotebookRestApi {
throws IOException, IllegalArgumentException {
LOG.info("stop note jobs {} ", noteId);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot stop this job for this note");
for (Paragraph p : note.getParagraphs()) {
if (!p.isTerminated()) {
@ -530,9 +604,8 @@ public class NotebookRestApi {
throws IOException, IllegalArgumentException {
LOG.info("get note job status.");
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status");
return new JsonResponse<>(Status.OK, null, note.generateParagraphsInfo()).build();
}
@ -553,14 +626,11 @@ public class NotebookRestApi {
throws IOException, IllegalArgumentException {
LOG.info("get note paragraph job status.");
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanRead(noteId, "Insufficient privileges you cannot get job status");
Paragraph paragraph = note.getParagraph(paragraphId);
if (paragraph == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
checkIfParagraphIsNotNull(paragraph);
return new JsonResponse<>(Status.OK, null, note.generateSingleParagraphInfo(paragraphId)).
build();
@ -583,14 +653,10 @@ public class NotebookRestApi {
LOG.info("run paragraph job asynchronously {} {} {}", noteId, paragraphId, message);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot run job for this note");
Paragraph paragraph = note.getParagraph(paragraphId);
if (paragraph == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
checkIfParagraphIsNotNull(paragraph);
// handle params if presented
handleParagraphParams(message, note, paragraph);
@ -625,14 +691,10 @@ public class NotebookRestApi {
LOG.info("run paragraph synchronously {} {} {}", noteId, paragraphId, message);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot run paragraph");
Paragraph paragraph = note.getParagraph(paragraphId);
if (paragraph == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
checkIfParagraphIsNotNull(paragraph);
// handle params if presented
handleParagraphParams(message, note, paragraph);
@ -667,14 +729,10 @@ public class NotebookRestApi {
@PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException {
LOG.info("stop paragraph job {} ", noteId);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot stop paragraph");
Paragraph p = note.getParagraph(paragraphId);
if (p == null) {
return new JsonResponse<>(Status.NOT_FOUND, "paragraph not found.").build();
}
checkIfParagraphIsNotNull(p);
p.abort();
return new JsonResponse<>(Status.OK).build();
}
@ -696,9 +754,8 @@ public class NotebookRestApi {
CronRequest request = gson.fromJson(message, CronRequest.class);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanWrite(noteId, "Insufficient privileges you cannot set a cron job for this note");
if (!CronExpression.isValidExpression(request.getCronString())) {
return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build();
@ -727,9 +784,9 @@ public class NotebookRestApi {
LOG.info("Remove cron job note {}", noteId);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserIsOwner(noteId,
"Insufficient privileges you cannot remove this cron job from this note");
Map<String, Object> config = note.getConfig();
config.put("cron", null);
@ -754,9 +811,8 @@ public class NotebookRestApi {
LOG.info("Get cron job note {}", noteId);
Note note = notebook.getNote(noteId);
if (note == null) {
return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build();
}
checkIfNoteIsNotNull(note);
checkIfUserCanRead(noteId, "Insufficient privileges you cannot get cron information");
return new JsonResponse<>(Status.OK, note.getConfig().get("cron")).build();
}

51
zeppelin-server/src/main/java/org/apache/zeppelin/rest/exception/ForbiddenException.java Normal file
View file

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.rest.exception;
import static javax.ws.rs.core.Response.Status.FORBIDDEN;
import static javax.ws.rs.core.Response.Status.UNAUTHORIZED;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import org.apache.zeppelin.utils.ExceptionUtils;
/**
* UnauthorizedException handler for WebApplicationException.
*
*/
public class ForbiddenException extends WebApplicationException {
private static final long serialVersionUID = 4394749068760407567L;
private static final String FORBIDDEN_MSG = "Not allowed to access";
public ForbiddenException() {
super(forbiddenJson(FORBIDDEN_MSG));
}
private static Response forbiddenJson(String message) {
return ExceptionUtils.jsonResponseContent(FORBIDDEN, message);
}
public ForbiddenException(Throwable cause, String message) {
super(cause, forbiddenJson(message));
}
public ForbiddenException(String message) {
super(forbiddenJson(message));
}
}

59
zeppelin-server/src/main/java/org/apache/zeppelin/rest/exception/NotFoundException.java Normal file
View file

@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.rest.exception;
import static javax.ws.rs.core.Response.Status.NOT_FOUND;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import org.apache.zeppelin.utils.ExceptionUtils;
/**
* Not Found handler for WebApplicationException.
*
*/
public class NotFoundException extends WebApplicationException {
private static final long serialVersionUID = 2459398393216512293L;
/**
* Create a HTTP 404 (Not Found) exception.
*/
public NotFoundException() {
super(ExceptionUtils.jsonResponse(NOT_FOUND));
}
/**
* Create a HTTP 404 (Not Found) exception.
* @param message the String that is the entity of the 404 response.
*/
public NotFoundException(String message) {
super(notFoundJson(message));
}
private static Response notFoundJson(String message) {
return ExceptionUtils.jsonResponseContent(NOT_FOUND, message);
}
public NotFoundException(Throwable cause) {
super(cause, notFoundJson(cause.getMessage()));
}
public NotFoundException(Throwable cause, String message) {
super(cause, notFoundJson(message));
}
}

33
zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/RestartInterpreterRequest.java Normal file
View file

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.rest.message;
/**
* RestartInterpreter rest api request message
*/
public class RestartInterpreterRequest {
String noteId;
public RestartInterpreterRequest() {
}
public String getNoteId() {
return noteId;
}
}

2
zeppelin-server/src/main/java/org/apache/zeppelin/server/ActiveDirectoryGroupRealm.java
View file

@ -304,7 +304,7 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm {
* @return a collection of roles that are implied by the given role names.
*/
protected Collection<String> getRoleNamesForGroups(Collection<String> groupNames) {
Set<String> roleNames = new HashSet<String>(groupNames.size());
Set<String> roleNames = new HashSet<>(groupNames.size());
if (groupRolesMap != null) {
for (String groupName : groupNames) {

2
zeppelin-server/src/main/java/org/apache/zeppelin/server/JsonResponse.java
View file

@ -75,7 +75,7 @@ public class JsonResponse<T> {
*/
public JsonResponse<T> addCookie(NewCookie newCookie) {
if (cookies == null) {
cookies = new ArrayList<NewCookie>();
cookies = new ArrayList<>();
}
cookies.add(newCookie);

2
zeppelin-server/src/main/java/org/apache/zeppelin/server/LdapGroupRealm.java
View file

@ -56,7 +56,7 @@ public class LdapGroupRealm extends JndiLdapRealm {
LdapContext ldapContext,
String userDnTemplate) throws NamingException {
try {
Set<String> roleNames = new LinkedHashSet<String>();
Set<String> roleNames = new LinkedHashSet<>();
SearchControls searchCtls = new SearchControls();
searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE);

21
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -26,7 +26,10 @@ import java.util.Set;
import javax.servlet.DispatcherType;
import javax.ws.rs.core.Application;
import org.apache.commons.lang.StringUtils;
import org.apache.cxf.jaxrs.servlet.CXFNonSpringJaxrsServlet;
import org.apache.shiro.web.env.EnvironmentLoaderListener;
import org.apache.shiro.web.servlet.ShiroFilter;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
@ -259,15 +262,13 @@ public class ZeppelinServer extends Application {
webapp.setSessionHandler(new SessionHandler());
webapp.addServlet(cxfServletHolder, "/api/*");
webapp.setInitParameter("shiroConfigLocations",
new File(conf.getShiroPath()).toURI().toString());
SecurityUtils.initSecurityManager(conf.getShiroPath());
webapp.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/api/*",
EnumSet.allOf(DispatcherType.class));
webapp.addEventListener(new org.apache.shiro.web.env.EnvironmentLoaderListener());
String shiroIniPath = conf.getShiroPath();
if (!StringUtils.isBlank(shiroIniPath)) {
webapp.setInitParameter("shiroConfigLocations", new File(shiroIniPath).toURI().toString());
SecurityUtils.initSecurityManager(shiroIniPath);
webapp.addFilter(ShiroFilter.class, "/api/*", EnumSet.allOf(DispatcherType.class));
webapp.addEventListener(new EnvironmentLoaderListener());
}
}
private static WebAppContext setupWebAppContext(ContextHandlerCollection contexts,
@ -302,7 +303,7 @@ public class ZeppelinServer extends Application {
@Override
public Set<Class<?>> getClasses() {
Set<Class<?>> classes = new HashSet<Class<?>>();
Set<Class<?>> classes = new HashSet<>();
return classes;
}

130
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -16,10 +16,22 @@
*/
package org.apache.zeppelin.socket;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@ -36,7 +48,13 @@ import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.NotebookEventListener;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.ParagraphJobListener;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
@ -54,13 +72,10 @@ import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
/**
* Zeppelin websocket service.
@ -86,8 +101,7 @@ public class NotebookServer extends WebSocketServlet implements
Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").create();
final Map<String, List<NotebookSocket>> noteSocketMap = new HashMap<>();
final Queue<NotebookSocket> connectedSockets = new ConcurrentLinkedQueue<>();
final Map<String, Queue<NotebookSocket>> userConnectedSockets =
new ConcurrentHashMap<String, Queue<NotebookSocket>>();
final Map<String, Queue<NotebookSocket>> userConnectedSockets = new ConcurrentHashMap<>();
private Notebook notebook() {
return ZeppelinServer.notebook;
@ -148,13 +162,12 @@ public class NotebookServer extends WebSocketServlet implements
}
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
boolean allowAnonymous = conf.
getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED);
boolean allowAnonymous = conf.isAnonymousAllowed();
if (!allowAnonymous && messagereceived.principal.equals("anonymous")) {
throw new Exception("Anonymous access not allowed ");
}
HashSet<String> userAndRoles = new HashSet<String>();
HashSet<String> userAndRoles = new HashSet<>();
userAndRoles.add(messagereceived.principal);
if (!messagereceived.roles.equals("")) {
HashSet<String> roles = gson.fromJson(messagereceived.roles,
@ -215,6 +228,9 @@ public class NotebookServer extends WebSocketServlet implements
case PARAGRAPH_CLEAR_OUTPUT:
clearParagraphOutput(conn, userAndRoles, notebook, messagereceived);
break;
case PARAGRAPH_CLEAR_ALL_OUTPUT:
clearAllParagraphOutput(conn, userAndRoles, notebook, messagereceived);
break;
case NOTE_UPDATE:
updateNote(conn, userAndRoles, notebook, messagereceived);
break;
@ -259,6 +275,9 @@ public class NotebookServer extends WebSocketServlet implements
case EDITOR_SETTING:
getEditorSetting(conn, messagereceived);
break;
case GET_INTERPRETER_SETTINGS:
getInterpreterSettings(conn, subject);
break;
default:
break;
}
@ -696,23 +715,49 @@ public class NotebookServer extends WebSocketServlet implements
return cronUpdated;
}
private void createNote(NotebookSocket conn, HashSet<String> userAndRoles,
Notebook notebook, Message message)
throws IOException {
AuthenticationInfo subject = new AuthenticationInfo(message.principal);
Note note = notebook.createNote(subject);
note.addParagraph(); // it's an empty note. so add one paragraph
if (message != null) {
String noteName = (String) message.get("name");
if (noteName == null || noteName.isEmpty()){
noteName = "Note " + note.getId();
}
note.setName(noteName);
}
note.persist(subject);
addConnectionToNote(note.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
try {
Note note = null;
String defaultInterpreterId = (String) message.get("defaultInterpreterId");
if (!StringUtils.isEmpty(defaultInterpreterId)) {
List<String> interpreterSettingIds = new LinkedList<>();
interpreterSettingIds.add(defaultInterpreterId);
for (String interpreterSettingId : notebook.getInterpreterFactory().
getDefaultInterpreterSettingList()) {
if (!interpreterSettingId.equals(defaultInterpreterId)) {
interpreterSettingIds.add(interpreterSettingId);
}
}
note = notebook.createNote(interpreterSettingIds, subject);
} else {
note = notebook.createNote(subject);
}
note.addParagraph(); // it's an empty note. so add one paragraph
if (message != null) {
String noteName = (String) message.get("name");
if (StringUtils.isEmpty(noteName)) {
noteName = "Note " + note.getId();
}
note.setName(noteName);
}
note.persist(subject);
addConnectionToNote(note.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
} catch (FileSystemException e) {
LOG.error("Exception from createNote", e);
conn.send(serializeMessage(new Message(OP.ERROR_INFO).put("info",
"Oops! There is something wrong with the notebook file system. "
+ "Please check the logs for more details.")));
return;
}
broadcastNoteList(subject, userAndRoles);
}
@ -780,6 +825,25 @@ public class NotebookServer extends WebSocketServlet implements
broadcastNoteList(subject, userAndRoles);
}
private void clearAllParagraphOutput(NotebookSocket conn, HashSet<String> userAndRoles,
Notebook notebook, Message fromMessage)
throws IOException {
final String noteId = (String) fromMessage.get("id");
if (StringUtils.isBlank(noteId)) {
return;
}
Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "clear output", fromMessage.principal,
userAndRoles, notebookAuthorization.getOwners(noteId));
return;
}
note.clearAllParagraphOutput();
broadcastNote(note);
}
protected Note importNote(NotebookSocket conn, HashSet<String> userAndRoles,
Notebook notebook, Message fromMessage)
throws IOException {
@ -1662,5 +1726,13 @@ public class NotebookServer extends WebSocketServlet implements
conn.send(serializeMessage(resp));
return;
}
private void getInterpreterSettings(NotebookSocket conn, AuthenticationInfo subject)
throws IOException {
List<InterpreterSetting> availableSettings = notebook().getInterpreterFactory().get();
conn.send(serializeMessage(new Message(OP.INTERPRETER_SETTINGS)
.put("interpreterSettings", availableSettings)));
}
}

36
zeppelin-server/src/main/java/org/apache/zeppelin/utils/ExceptionUtils.java Normal file
View file

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.utils;
import javax.ws.rs.core.Response.Status;
import org.apache.zeppelin.server.JsonResponse;
/**
* Utility method for exception in rest api.
*
*/
public class ExceptionUtils {
public static javax.ws.rs.core.Response jsonResponse(Status status) {
return new JsonResponse<>(status).build();
}
public static javax.ws.rs.core.Response jsonResponseContent(Status status, String message) {
return new JsonResponse<>(status, message).build();
}
}

39
zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
View file

@ -16,30 +16,41 @@
*/
package org.apache.zeppelin.utils;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import org.apache.shiro.config.IniSecurityManagerFactory;
import org.apache.shiro.mgt.SecurityManager;
import org.apache.shiro.realm.Realm;
import org.apache.shiro.realm.text.IniRealm;
import org.apache.shiro.subject.Subject;
import org.apache.shiro.util.ThreadContext;
import org.apache.shiro.web.mgt.DefaultWebSecurityManager;
import org.apache.shiro.mgt.SecurityManager;
import org.apache.shiro.config.IniSecurityManagerFactory;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.*;
import com.google.common.collect.Sets;
/**
* Tools for securing Zeppelin
*/
public class SecurityUtils {
private static final String ANONYMOUS = "anonymous";
private static final HashSet<String> EMPTY_HASHSET = Sets.newHashSet();
private static boolean isEnabled = false;
public static void initSecurityManager(String shiroPath) {
IniSecurityManagerFactory factory = new IniSecurityManagerFactory("file:" + shiroPath);
SecurityManager securityManager = factory.getInstance();
org.apache.shiro.SecurityUtils.setSecurityManager(securityManager);
isEnabled = true;
}
public static Boolean isValidOrigin(String sourceHost, ZeppelinConfiguration conf)
@ -65,18 +76,24 @@ public class SecurityUtils {
* @return shiro principal
*/
public static String getPrincipal() {
if (!isEnabled) {
return ANONYMOUS;
}
Subject subject = org.apache.shiro.SecurityUtils.getSubject();
String principal;
if (subject.isAuthenticated()) {
principal = subject.getPrincipal().toString();
} else {
principal = "anonymous";
principal = ANONYMOUS;
}
return principal;
}
public static Collection getRealmsList() {
if (!isEnabled) {
return Collections.emptyList();
}
DefaultWebSecurityManager defaultWebSecurityManager;
String key = ThreadContext.SECURITY_MANAGER_KEY;
defaultWebSecurityManager = (DefaultWebSecurityManager) ThreadContext.get(key);
@ -91,6 +108,9 @@ public class SecurityUtils {
* @return shiro roles
*/
public static HashSet<String> getRoles() {
if (!isEnabled) {
return EMPTY_HASHSET;
}
Subject subject = org.apache.shiro.SecurityUtils.getSubject();
HashSet<String> roles = new HashSet<>();
Map allRoles = null;
@ -123,6 +143,9 @@ public class SecurityUtils {
* Checked if shiro enabled or not
*/
public static boolean isAuthenticated() {
if (!isEnabled) {
return false;
}
return org.apache.shiro.SecurityUtils.getSubject().isAuthenticated();
}
}

6
zeppelin-server/src/test/java/com/webautomation/ScreenCaptureHtmlUnitDriver.java
View file

@ -114,8 +114,8 @@ public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements Takes
// http://stackoverflow.com/questions/2244272/how-can-i-tell-htmlunits-webclient-to-download-images-and-css
protected byte[] downloadCssAndImages(WebClient webClient, HtmlPage page) throws Exception {
WebWindow currentWindow = webClient.getCurrentWindow();
Map<String, String> urlMapping = new HashMap<String, String>();
Map<String, byte[]> files = new HashMap<String, byte[]>();
Map<String, String> urlMapping = new HashMap<>();
Map<String, byte[]> files = new HashMap<>();
WebWindow window = null;
try {
window = webClient.getWebWindowByName(page.getUrl().toString()+"_screenshot");
@ -199,7 +199,7 @@ public class ScreenCaptureHtmlUnitDriver extends HtmlUnitDriver implements Takes
}
List<String> getLinksFromCss(String css) {
List<String> result = new LinkedList<String>();
List<String> result = new LinkedList<>();
Matcher m = cssUrlPattern.matcher(css);
while (m.find()) { // find next match
result.add( m.group(2));

2
zeppelin-server/src/test/java/org/apache/zeppelin/AbstractZeppelinIT.java
View file

@ -88,7 +88,7 @@ abstract public class AbstractZeppelinIT {
}
protected WebElement pollingWait(final By locator, final long timeWait) {
Wait<WebDriver> wait = new FluentWait<WebDriver>(driver)
Wait<WebDriver> wait = new FluentWait<>(driver)
.withTimeout(timeWait, TimeUnit.SECONDS)
.pollingEvery(1, TimeUnit.SECONDS)
.ignoring(NoSuchElementException.class);

2
zeppelin-server/src/test/java/org/apache/zeppelin/CommandExecutor.java
View file

@ -40,7 +40,7 @@ public class CommandExecutor {
private static IGNORE_ERRORS DEFAULT_BEHAVIOUR_ON_ERRORS = IGNORE_ERRORS.TRUE;
public static Object executeCommandLocalHost(String[] command, boolean printToConsole, ProcessData.Types_Of_Data type, IGNORE_ERRORS ignore_errors) {
List<String> subCommandsAsList = new ArrayList<String>(Arrays.asList(command));
List<String> subCommandsAsList = new ArrayList<>(Arrays.asList(command));
String mergedCommand = StringUtils.join(subCommandsAsList, " ");
LOG.info("Sending command \"" + mergedCommand + "\" to localhost");

51
zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
View file

@ -16,6 +16,13 @@
*/
package org.apache.zeppelin.integration;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.AbstractZeppelinIT;
@ -34,11 +41,6 @@ import org.openqa.selenium.WebElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.List;
/**
* Created for org.apache.zeppelin.integration on 13/06/16.
@ -48,7 +50,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
@Rule
public ErrorCollector collector = new ErrorCollector();
static String shiroPath;
static String authShiro = "[users]\n" +
"admin = password1, admin\n" +
"finance1 = finance1, finance\n" +
@ -80,8 +82,11 @@ public class AuthenticationIT extends AbstractZeppelinIT {
try {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), "../");
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
File file = new File(conf.getShiroPath());
originalShiro = StringUtils.join(FileUtils.readLines(file, "UTF-8"), "\n");
shiroPath = conf.getRelativeDir(String.format("%s/shiro.ini", conf.getConfDir()));
File file = new File(shiroPath);
if (file.exists()) {
originalShiro = StringUtils.join(FileUtils.readLines(file, "UTF-8"), "\n");
}
FileUtils.write(file, authShiro, "UTF-8");
} catch (IOException e) {
LOG.error("Error in AuthenticationIT startUp::", e);
@ -97,9 +102,14 @@ public class AuthenticationIT extends AbstractZeppelinIT {
return;
}
try {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
File file = new File(conf.getShiroPath());
FileUtils.write(file, originalShiro, "UTF-8");
if (!StringUtils.isBlank(shiroPath)) {
File file = new File(shiroPath);
if (StringUtils.isBlank(originalShiro)) {
FileUtils.deleteQuietly(file);
} else {
FileUtils.write(file, originalShiro, "UTF-8");
}
}
} catch (IOException e) {
LOG.error("Error in AuthenticationIT tearDown::", e);
}
@ -119,6 +129,23 @@ public class AuthenticationIT extends AbstractZeppelinIT {
ZeppelinITUtils.sleep(1000, false);
}
private void testShowNotebookListOnNavbar() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
pollingWait(By.xpath("//li[@class='dropdown notebook-list-dropdown']"),
MAX_BROWSER_TIMEOUT_SEC).click();
assertTrue(driver.findElements(By.xpath("//a[@class=\"notebook-list-item ng-scope\"]")).size() > 0);
pollingWait(By.xpath("//li[@class='dropdown notebook-list-dropdown']"),
MAX_BROWSER_TIMEOUT_SEC).click();
pollingWait(By.xpath("//li[@class='dropdown notebook-list-dropdown']"),
MAX_BROWSER_TIMEOUT_SEC).click();
} catch (Exception e) {
handleException("Exception in ParagraphActionsIT while testShowNotebookListOnNavbar ", e);
}
}
private void logoutUser(String userName) {
ZeppelinITUtils.sleep(500, false);
driver.findElement(By.xpath("//div[contains(@class, 'navbar-collapse')]//li[contains(.,'" +
@ -144,7 +171,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
authenticationIT.logoutUser("admin");
} catch (Exception e) {
handleException("Exception in ParagraphActionsIT while testCreateNewButton ", e);
handleException("Exception in AuthenticationIT while testCreateNewButton ", e);
}
}

82
zeppelin-server/src/test/java/org/apache/zeppelin/integration/InterpreterIT.java Normal file
View file

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.integration;
import org.apache.zeppelin.AbstractZeppelinIT;
import org.apache.zeppelin.WebDriverManager;
import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import org.openqa.selenium.By;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.Select;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class InterpreterIT extends AbstractZeppelinIT {
private static final Logger LOG = LoggerFactory.getLogger(InterpreterIT.class);
@Rule
public ErrorCollector collector = new ErrorCollector();
@Before
public void startUp() {
if (!endToEndTestEnabled()) {
return;
}
driver = WebDriverManager.getWebDriver();
}
@After
public void tearDown() {
if (!endToEndTestEnabled()) {
return;
}
driver.quit();
}
@Test
public void testShowDescriptionOnInterpreterCreate() throws Exception {
if (!endToEndTestEnabled()) {
return;
}
try {
// navigate to interpreter page
WebElement settingButton = driver.findElement(By.xpath("//button[@class='nav-btn dropdown-toggle ng-scope']"));
settingButton.click();
WebElement interpreterLink = driver.findElement(By.xpath("//a[@href='#/interpreter']"));
interpreterLink.click();
WebElement createButton = driver.findElement(By.xpath("//button[contains(., 'Create')]"));
createButton.click();
Select select = new Select(driver.findElement(By.xpath("//select[@ng-change='newInterpreterGroupChange()']")));
select.selectByVisibleText("spark");
collector.checkThat("description of interpreter property is displayed",
driver.findElement(By.xpath("//tr/td[contains(text(), 'spark.app.name')]/following-sibling::td[2]")).getText(),
CoreMatchers.equalTo("The name of spark application."));
} catch (Exception e) {
handleException("Exception in InterpreterIT while testShowDescriptionOnInterpreterCreate ", e);
}
}
}

2
zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
View file

@ -29,7 +29,7 @@ import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
public class MockInterpreter1 extends Interpreter{
Map<String, Object> vars = new HashMap<String, Object>();
Map<String, Object> vars = new HashMap<>();
public MockInterpreter1(Properties property) {
super(property);

166
zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
View file

@ -23,20 +23,25 @@ import java.lang.ref.WeakReference;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Pattern;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.ByteArrayRequestEntity;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.server.ZeppelinServer;
@ -46,6 +51,7 @@ import org.hamcrest.TypeSafeMatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
import com.google.gson.JsonParser;
@ -59,6 +65,29 @@ public abstract class AbstractTestRestApi {
protected static final boolean wasRunning = checkIfServerIsRunning();
static boolean pySpark = false;
static boolean sparkR = false;
static Gson gson = new Gson();
static boolean isRunningWithAuth = false;
private static File shiroIni = null;
private static String zeppelinShiro =
"[users]\n" +
"admin = password1, admin\n" +
"user1 = password2, role1, role2\n" +
"user2 = password3, role3\n" +
"user3 = password4, role2\n" +
"[main]\n" +
"sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager\n" +
"securityManager.sessionManager = $sessionManager\n" +
"securityManager.sessionManager.globalSessionTimeout = 86400000\n" +
"shiro.loginUrl = /api/login\n" +
"[roles]\n" +
"role1 = *\n" +
"role2 = *\n" +
"role3 = *\n" +
"admin = *" +
"[urls]\n" +
"/api/version = anon\n" +
"/** = authc";
private String getUrl(String path) {
String url;
@ -94,15 +123,27 @@ public abstract class AbstractTestRestApi {
}
};
protected static void startUp() throws Exception {
private static void start(boolean withAuth) throws Exception {
if (!wasRunning) {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), "../");
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_WAR.getVarName(), "../zeppelin-web/dist");
LOG.info("Staring test Zeppelin up...");
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
if (withAuth) {
isRunningWithAuth = true;
// Set Anonymous session to false.
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED.getVarName(), "false");
// Create a shiro env test.
shiroIni = new File("../conf/shiro.ini");
if (!shiroIni.exists()) {
shiroIni.createNewFile();
}
FileUtils.writeStringToFile(shiroIni, zeppelinShiro);
}
// exclude org.apache.zeppelin.rinterpreter.* for scala 2.11 test
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
String interpreters = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS);
String interpretersCompatibleWithScala211Test = null;
@ -140,47 +181,41 @@ public abstract class AbstractTestRestApi {
LOG.info("Test Zeppelin stared.");
// assume first one is spark
InterpreterSetting sparkIntpSetting = null;
for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
if (intpSetting.getName().equals("spark")) {
sparkIntpSetting = intpSetting;
}
}
Properties sparkProperties = (Properties) sparkIntpSetting.getProperties();
// ci environment runs spark cluster for testing
// so configure zeppelin use spark cluster
if ("true".equals(System.getenv("CI"))) {
// assume first one is spark
InterpreterSetting sparkIntpSetting = null;
for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
if (intpSetting.getName().equals("spark")) {
sparkIntpSetting = intpSetting;
}
}
// set spark master and other properties
sparkIntpSetting.getProperties().setProperty("master", "local[2]");
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false");
sparkProperties.setProperty("master", "local[2]");
sparkProperties.setProperty("spark.cores.max", "2");
sparkProperties.setProperty("zeppelin.spark.useHiveContext", "false");
// set spark home for pyspark
sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome());
sparkProperties.setProperty("spark.home", getSparkHome());
sparkIntpSetting.setProperties(sparkProperties);
pySpark = true;
sparkR = true;
ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId());
} else {
// assume first one is spark
InterpreterSetting sparkIntpSetting = null;
for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
if (intpSetting.getName().equals("spark")) {
sparkIntpSetting = intpSetting;
}
}
String sparkHome = getSparkHome();
if (sparkHome != null) {
if (System.getenv("SPARK_MASTER") != null) {
sparkIntpSetting.getProperties().setProperty("master", System.getenv("SPARK_MASTER"));
sparkProperties.setProperty("master", System.getenv("SPARK_MASTER"));
} else {
sparkIntpSetting.getProperties()
.setProperty("master", "local[2]");
sparkProperties.setProperty("master", "local[2]");
}
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
sparkProperties.setProperty("spark.cores.max", "2");
// set spark home for pyspark
sparkIntpSetting.getProperties().setProperty("spark.home", sparkHome);
sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false");
sparkProperties.setProperty("spark.home", sparkHome);
sparkProperties.setProperty("zeppelin.spark.useHiveContext", "false");
pySpark = true;
sparkR = true;
}
@ -189,6 +224,14 @@ public abstract class AbstractTestRestApi {
}
}
}
protected static void startUpWithAuthenticationEnable() throws Exception {
start(true);
}
protected static void startUp() throws Exception {
start(false);
}
private static String getHostname() {
try {
@ -249,7 +292,9 @@ public abstract class AbstractTestRestApi {
for (String setting : settingList) {
ZeppelinServer.notebook.getInterpreterFactory().restart(setting);
}
if (shiroIni != null) {
FileUtils.deleteQuietly(shiroIni);
}
LOG.info("Terminating test Zeppelin...");
ZeppelinServer.jettyWebServer.stop();
executor.shutdown();
@ -270,6 +315,11 @@ public abstract class AbstractTestRestApi {
LOG.info("Test Zeppelin terminated.");
System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName());
if (isRunningWithAuth) {
isRunningWithAuth = false;
System
.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED.getVarName());
}
}
}
@ -291,56 +341,104 @@ public abstract class AbstractTestRestApi {
}
protected static GetMethod httpGet(String path) throws IOException {
return httpGet(path, StringUtils.EMPTY, StringUtils.EMPTY);
}
protected static GetMethod httpGet(String path, String user, String pwd) throws IOException {
LOG.info("Connecting to {}", url + path);
HttpClient httpClient = new HttpClient();
GetMethod getMethod = new GetMethod(url + path);
getMethod.addRequestHeader("Origin", url);
if (userAndPasswordAreNotBlank(user, pwd)) {
getMethod.setRequestHeader("Cookie", "JSESSIONID="+ getCookie(user, pwd));
}
httpClient.executeMethod(getMethod);
LOG.info("{} - {}", getMethod.getStatusCode(), getMethod.getStatusText());
return getMethod;
}
protected static DeleteMethod httpDelete(String path) throws IOException {
return httpDelete(path, StringUtils.EMPTY, StringUtils.EMPTY);
}
protected static DeleteMethod httpDelete(String path, String user, String pwd) throws IOException {
LOG.info("Connecting to {}", url + path);
HttpClient httpClient = new HttpClient();
DeleteMethod deleteMethod = new DeleteMethod(url + path);
deleteMethod.addRequestHeader("Origin", url);
if (userAndPasswordAreNotBlank(user, pwd)) {
deleteMethod.setRequestHeader("Cookie", "JSESSIONID="+ getCookie(user, pwd));
}
httpClient.executeMethod(deleteMethod);
LOG.info("{} - {}", deleteMethod.getStatusCode(), deleteMethod.getStatusText());
return deleteMethod;
}
protected static PostMethod httpPost(String path, String body) throws IOException {
return httpPost(path, body, StringUtils.EMPTY, StringUtils.EMPTY);
}
protected static PostMethod httpPost(String path, String request, String user, String pwd)
throws IOException {
LOG.info("Connecting to {}", url + path);
HttpClient httpClient = new HttpClient();
PostMethod postMethod = new PostMethod(url + path);
postMethod.addRequestHeader("Origin", url);
RequestEntity entity = new ByteArrayRequestEntity(body.getBytes("UTF-8"));
postMethod.setRequestEntity(entity);
postMethod.setRequestBody(request);
postMethod.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
if (userAndPasswordAreNotBlank(user, pwd)) {
postMethod.setRequestHeader("Cookie", "JSESSIONID="+ getCookie(user, pwd));
}
httpClient.executeMethod(postMethod);
LOG.info("{} - {}", postMethod.getStatusCode(), postMethod.getStatusText());
return postMethod;
}
protected static PutMethod httpPut(String path, String body) throws IOException {
return httpPut(path, body, StringUtils.EMPTY, StringUtils.EMPTY);
}
protected static PutMethod httpPut(String path, String body, String user, String pwd) throws IOException {
LOG.info("Connecting to {}", url + path);
HttpClient httpClient = new HttpClient();
PutMethod putMethod = new PutMethod(url + path);
putMethod.addRequestHeader("Origin", url);
RequestEntity entity = new ByteArrayRequestEntity(body.getBytes("UTF-8"));
putMethod.setRequestEntity(entity);
if (userAndPasswordAreNotBlank(user, pwd)) {
putMethod.setRequestHeader("Cookie", "JSESSIONID="+ getCookie(user, pwd));
}
httpClient.executeMethod(putMethod);
LOG.info("{} - {}", putMethod.getStatusCode(), putMethod.getStatusText());
return putMethod;
}
private static String getCookie(String user, String password) throws IOException {
HttpClient httpClient = new HttpClient();
PostMethod postMethod = new PostMethod(url + "/login");
postMethod.addRequestHeader("Origin", url);
postMethod.setParameter("password", password);
postMethod.setParameter("userName", user);
httpClient.executeMethod(postMethod);
LOG.info("{} - {}", postMethod.getStatusCode(), postMethod.getStatusText());
Pattern pattern = Pattern.compile("JSESSIONID=([a-zA-Z0-9-]*)");
java.util.regex.Matcher matcher = pattern.matcher(postMethod.getResponseHeaders("Set-Cookie")[0].toString());
return matcher.find()? matcher.group(1) : StringUtils.EMPTY;
}
protected static boolean userAndPasswordAreNotBlank(String user, String pwd) {
if (StringUtils.isBlank(user) && StringUtils.isBlank(pwd)) {
return false;
}
return true;
}
protected Matcher<HttpMethodBase> responsesWith(final int expectedStatusCode) {
return new TypeSafeMatcher<HttpMethodBase>() {
WeakReference<HttpMethodBase> method;
@Override
public boolean matchesSafely(HttpMethodBase httpMethodBase) {
method = (method == null) ? new WeakReference<HttpMethodBase>(httpMethodBase) : method;
method = (method == null) ? new WeakReference<>(httpMethodBase) : method;
return httpMethodBase.getStatusCode() == expectedStatusCode;
}
@ -441,7 +539,7 @@ public abstract class AbstractTestRestApi {
/** Status code matcher */
protected Matcher<? super HttpMethodBase> isForbiden() { return responsesWith(403); }
protected Matcher<? super HttpMethodBase> isForbidden() { return responsesWith(403); }
protected Matcher<? super HttpMethodBase> isAllowed() {
return responsesWith(200);

52
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -25,6 +25,7 @@ import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
@ -200,6 +201,57 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRestartInterpreterPerNote() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
config.put("enabled", true);
// run markdown paragraph.
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown</p>\n", p.getResult().message());
// get md interpreter
InterpreterSetting mdIntpSetting = null;
for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) {
if (setting.getName().equals("md")) {
mdIntpSetting = setting;
break;
}
}
String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";
// Restart isolated mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("isolated interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart scoped mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("scoped interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart shared mode of Interpreter for note.
mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("shared interpreter restart:", put, isAllowed());
put.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testListRepository() throws IOException {
GetMethod get = httpGet("/interpreter/repository");

45
zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
View file

@ -18,15 +18,14 @@
package org.apache.zeppelin.rest;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.NotebookAuthorizationInfoSaving;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
@ -37,12 +36,11 @@ import org.junit.Test;
import org.junit.runners.MethodSorters;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
/**
@ -179,6 +177,43 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
ZeppelinServer.notebook.removeNote(clonedNoteId, anonymous);
}
@Test
public void testClearAllParagraphOutput() throws IOException {
// Create note and set result explicitly
Note note = ZeppelinServer.notebook.createNote(anonymous);
Paragraph p1 = note.addParagraph();
InterpreterResult result = new InterpreterResult(InterpreterResult.Code.SUCCESS, InterpreterResult.Type.TEXT, "result");
p1.setResult(result);
Paragraph p2 = note.addParagraph();
p2.setReturn(result, new Throwable());
// clear paragraph result
PutMethod put = httpPut("/notebook/" + note.getId() + "/clear", "");
LOG.info("test clear paragraph output response\n" + put.getResponseBodyAsString());
assertThat(put, isAllowed());
put.releaseConnection();
// check if paragraph results are cleared
GetMethod get = httpGet("/notebook/" + note.getId() + "/paragraph/" + p1.getId());
assertThat(get, isAllowed());
Map<String, Object> resp1 = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, Object> resp1Body = (Map<String, Object>) resp1.get("body");
assertNull(resp1Body.get("result"));
get = httpGet("/notebook/" + note.getId() + "/paragraph/" + p2.getId());
assertThat(get, isAllowed());
Map<String, Object> resp2 = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
Map<String, Object> resp2Body = (Map<String, Object>) resp2.get("body");
assertNull(resp2Body.get("result"));
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
}

156
zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookSecurityRestApiTest.java Normal file
View file

@ -0,0 +1,156 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.rest;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import java.io.IOException;
import java.util.Map;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.server.ZeppelinServer;
import org.hamcrest.Matcher;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
public class NotebookSecurityRestApiTest extends AbstractTestRestApi {
Gson gson = new Gson();
@BeforeClass
public static void init() throws Exception {
AbstractTestRestApi.startUpWithAuthenticationEnable();
}
@AfterClass
public static void destroy() throws Exception {
AbstractTestRestApi.shutDown();
}
@Before
public void setUp() {}
@Test
public void testThatUserCanCreateAndRemoveNote() throws IOException {
String noteId = createNoteForUser("test", "admin", "password1");
assertNotNull(noteId);
String id = getNoteIdForUser(noteId, "admin", "password1");
assertThat(id, is(noteId));
deleteNoteForUser(noteId, "admin", "password1");
}
@Test
public void testThatOtherUserCanAccessNoteIfPermissionNotSet() throws IOException {
String noteId = createNoteForUser("test", "admin", "password1");
userTryGetNote(noteId, "user1", "password2", isAllowed());
deleteNoteForUser(noteId, "admin", "password1");
}
@Test
public void testThatOtherUserCannotAccessNoteIfPermissionSet() throws IOException {
String noteId = createNoteForUser("test", "admin", "password1");
//set permission
String payload = "{ \"owners\": [\"admin\"], \"readers\": [\"user2\"], \"writers\": [\"user2\"] }";
PutMethod put = httpPut("/notebook/" + noteId + "/permissions", payload , "admin", "password1");
assertThat("test set note permission method:", put, isAllowed());
put.releaseConnection();
userTryGetNote(noteId, "user1", "password2", isForbidden());
userTryGetNote(noteId, "user2", "password3", isAllowed());
deleteNoteForUser(noteId, "admin", "password1");
}
@Test
public void testThatWriterCannotRemoveNote() throws IOException {
String noteId = createNoteForUser("test", "admin", "password1");
//set permission
String payload = "{ \"owners\": [\"admin\", \"user1\"], \"readers\": [\"user2\"], \"writers\": [\"user2\"] }";
PutMethod put = httpPut("/notebook/" + noteId + "/permissions", payload , "admin", "password1");
assertThat("test set note permission method:", put, isAllowed());
put.releaseConnection();
userTryRemoveNote(noteId, "user2", "password3", isForbidden());
userTryRemoveNote(noteId, "user1", "password2", isAllowed());
Note deletedNote = ZeppelinServer.notebook.getNote(noteId);
assertNull("Deleted note should be null", deletedNote);
}
private void userTryRemoveNote(String noteId, String user, String pwd, Matcher<? super HttpMethodBase> m) throws IOException {
DeleteMethod delete = httpDelete(("/notebook/" + noteId), user, pwd);
assertThat(delete, m);
delete.releaseConnection();
}
private void userTryGetNote(String noteId, String user, String pwd, Matcher<? super HttpMethodBase> m) throws IOException {
GetMethod get = httpGet("/notebook/" + noteId, user, pwd);
assertThat(get, m);
get.releaseConnection();
}
private String getNoteIdForUser(String noteId, String user, String pwd) throws IOException {
GetMethod get = httpGet("/notebook/" + noteId, user, pwd);
assertThat("test note create method:", get, isAllowed());
Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
get.releaseConnection();
return (String) ((Map<String, Object>)resp.get("body")).get("id");
}
private String createNoteForUser(String noteName, String user, String pwd) throws IOException {
String jsonRequest = "{\"name\":\"" + noteName + "\"}";
PostMethod post = httpPost("/notebook/", jsonRequest, user, pwd);
assertThat("test note create method:", post, isCreated());
Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
}.getType());
post.releaseConnection();
String newNoteId = (String) resp.get("body");
Note newNote = ZeppelinServer.notebook.getNote(newNoteId);
assertNotNull("Can not find new note by id", newNote);
return newNoteId;
}
private void deleteNoteForUser(String noteId, String user, String pwd) throws IOException {
DeleteMethod delete = httpDelete(("/notebook/" + noteId), user, pwd);
assertThat("Test delete method:", delete, isAllowed());
delete.releaseConnection();
// make sure note is deleted
if (!noteId.isEmpty()) {
Note deletedNote = ZeppelinServer.notebook.getNote(noteId);
assertNull("Deleted note should be null", deletedNote);
}
}
}

15
zeppelin-server/src/test/java/org/apache/zeppelin/rest/SecurityRestApiTest.java
View file

@ -17,8 +17,10 @@
package org.apache.zeppelin.rest;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.commons.httpclient.methods.GetMethod;
import org.hamcrest.CoreMatchers;
import org.junit.AfterClass;
@ -27,11 +29,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ErrorCollector;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.*;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
public class SecurityRestApiTest extends AbstractTestRestApi {
Gson gson = new Gson();
@ -41,7 +40,7 @@ public class SecurityRestApiTest extends AbstractTestRestApi {
@BeforeClass
public static void init() throws Exception {
AbstractTestRestApi.startUp();
AbstractTestRestApi.startUpWithAuthenticationEnable();;
}
@AfterClass

42
zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
View file

@ -360,6 +360,48 @@ public class NotebookServerTest extends AbstractTestRestApi {
verify(otherConn).send(mdMsg1);
}
@Test
public void testCreateNoteWithDefaultInterpreterId() throws IOException {
// create two sockets and open it
NotebookSocket sock1 = createWebSocket();
NotebookSocket sock2 = createWebSocket();
assertEquals(sock1, sock1);
assertNotEquals(sock1, sock2);
notebookServer.onOpen(sock1);
notebookServer.onOpen(sock2);
String noteName = "Note with millis " + System.currentTimeMillis();
String defaultInterpreterId = "";
List<InterpreterSetting> settings = notebook.getInterpreterFactory().get();
if (settings.size() > 1) {
defaultInterpreterId = settings.get(1).getId();
}
// create note from sock1
notebookServer.onMessage(sock1, gson.toJson(
new Message(OP.NEW_NOTE)
.put("name", noteName)
.put("defaultInterpreterId", defaultInterpreterId)));
// expect the events are broadcasted properly
verify(sock1, times(2)).send(anyString());
Note createdNote = null;
for (Note note : notebook.getAllNotes()) {
if (note.getName().equals(noteName)) {
createdNote = note;
break;
}
}
if (settings.size() > 1) {
assertEquals(notebook.getInterpreterFactory().getDefaultInterpreterSetting(
createdNote.getId()).getId(), defaultInterpreterId);
}
notebook.removeNote(createdNote.getId(), anonymous);
}
private NotebookSocket createWebSocket() {
NotebookSocket sock = mock(NotebookSocket.class);
when(sock.getRequest()).thenReturn(mockRequest);

15
zeppelin-web/src/app/home/home.controller.js
View file

@ -22,10 +22,12 @@
'websocketMsgSrv',
'$rootScope',
'arrayOrderingSrv',
'ngToast'
'ngToast',
'noteActionSrv'
];
function HomeCtrl($scope, noteListDataFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv, ngToast) {
function HomeCtrl($scope, noteListDataFactory, websocketMsgSrv, $rootScope, arrayOrderingSrv,
ngToast, noteActionSrv) {
ngToast.dismiss();
var vm = this;
vm.notes = noteListDataFactory;
@ -85,6 +87,13 @@
vm.notebookHome = false;
}
});
}
$scope.removeNote = function(noteId) {
noteActionSrv.removeNote(noteId, false);
};
$scope.clearAllParagraphOutput = function(noteId) {
noteActionSrv.clearAllParagraphOutput(noteId);
};
}
})();

16
zeppelin-web/src/app/home/home.html
View file

@ -13,10 +13,24 @@ limitations under the License.
-->
<script type="text/ng-template" id="notebook_folder_renderer.html">
<div ng-if="node.children == null">
<div ng-if="node.children == null"
ng-mouseenter="showButton=true"
ng-mouseleave="showButton=false">
<a style="text-decoration: none;" href="#/notebook/{{node.id}}">
<i style="font-size: 10px;" class="icon-doc"/> {{noteName(node)}}
</a>
<a style="text-decoration: none;">
<i style="font-size: 13px; margin-left: 10px; cursor: pointer; text-decoration: none;"
class="fa fa-eraser" ng-show="showButton" ng-click="clearAllParagraphOutput(node.id)"
tooltip-placement="bottom" tooltip="Clear output">
</i>
</a>
<a style="text-decoration: none;">
<i style="font-size: 13px; margin-left: 2px; cursor: pointer; text-decoration: none;"
class="fa fa-trash-o" ng-show="showButton" ng-click="removeNote(node.id)"
tooltip-placement="bottom" tooltip="Remove note">
</i>
</a>
</div>
<div ng-if="node.children != null">
<a style="text-decoration: none; cursor: pointer;" ng-click="toggleFolderNode(node)">

6
zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
View file

@ -263,10 +263,10 @@ limitations under the License.
<th>description</th>
<th>action</th>
</tr>
<tr ng-repeat="(key, value) in newInterpreterSetting.properties">
<tr ng-repeat="key in newInterpreterSetting.properties | sortByKey">
<td>{{key}}</td>
<td><textarea msd-elastic ng-model="value.value"></textarea></td>
<td>{{value.description}}</td>
<td><textarea msd-elastic ng-model="newInterpreterSetting.properties[key].value"></textarea></td>
<td>{{newInterpreterSetting.properties[key].description}}</td>
<td>
<button class="btn btn-default btn-sm fa fa-remove" ng-click="removeInterpreterProperty(key)">
</button>

3
zeppelin-web/src/app/interpreter/interpreter.controller.js
View file

@ -399,12 +399,11 @@
var intpInfo = el[i];
for (var key in intpInfo) {
properties[key] = {
value: intpInfo[key],
value: intpInfo[key].defaultValue,
description: intpInfo[key].description
};
}
}
$scope.newInterpreterSetting.properties = properties;
};

2
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -43,7 +43,7 @@ limitations under the License.
</button>
<button type="button"
class="btn btn-default btn-xs"
ng-click="clearAllParagraphOutput()"
ng-click="clearAllParagraphOutput(note.id)"
ng-hide="viewOnly"
ng-class="{'disabled':isNoteRunning()}"
tooltip-placement="bottom" tooltip="Clear output">

Some files were not shown because too many files have changed in this diff Show more