Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

rebase and fixing testcase

Browse source
This commit is contained in:
astroshim 2016-10-21 16:55:06 +09:00
commit a626188490
39 changed files with 1196 additions and 655 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
spark-dependencies/pom.xml
View file

@ -525,9 +525,9 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>2.0.0</spark.version>
<spark.version>2.0.1</spark.version>
<protobuf.version>2.5.0</protobuf.version>
<py4j.version>0.10.1</py4j.version>
<py4j.version>0.10.3</py4j.version>
<scala.version>2.11.8</scala.version>
</properties>
</profile>

6
spark/pom.xml
View file

@ -37,7 +37,7 @@
<jsoup.version>1.8.2</jsoup.version>
<mockito.version>1.10.19</mockito.version>
<powermock.version>1.6.4</powermock.version>
<spark.version>2.0.0</spark.version>
<spark.version>2.0.1</spark.version>
</properties>
<dependencies>
@ -519,9 +519,9 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<spark.version>2.0.0</spark.version>
<spark.version>2.0.1</spark.version>
<protobuf.version>2.5.0</protobuf.version>
<py4j.version>0.10.1</py4j.version>
<py4j.version>0.10.3</py4j.version>
<scala.version>2.11.8</scala.version>
</properties>
</profile>

2
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -481,7 +481,7 @@ public class SparkInterpreter extends Interpreter {
//Only one of py4j-0.9-src.zip and py4j-0.8.2.1-src.zip should exist
String[] pythonLibs = new String[]{"pyspark.zip", "py4j-0.9-src.zip", "py4j-0.8.2.1-src.zip",
"py4j-0.10.1-src.zip"};
"py4j-0.10.1-src.zip", "py4j-0.10.3-src.zip"};
ArrayList<String> pythonLibUris = new ArrayList<>();
for (String lib : pythonLibs) {
File libFile = new File(pysparkPath, lib);

3
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java
View file

@ -141,6 +141,9 @@ public class ZeppelinR implements ExecuteResultHandler {
cmd.addArgument(Integer.toString(port));
cmd.addArgument(libPath);
cmd.addArgument(Integer.toString(sparkVersion.toNumber()));
// dump out the R command to facilitate manually running it, e.g. for fault diagnosis purposes
logger.debug(cmd.toString());
executor = new DefaultExecutor();
outputStream = new SparkOutputStream(logger);

2
zeppelin-distribution/src/bin_license/LICENSE
View file

@ -259,7 +259,7 @@ The following components are provided under the BSD-style License.
(BSD-like) ASM asm-utils (org.ow2.asm:asm-utils:5.0.3 - http://asm.ow2.org/) - Copyright (c) 2000-2011 INRIA, France Telecom
(New BSD License) Markdown4j (org.commonjava.googlecode.markdown4j:markdown4j:jar:2.2-cj-1.0 - https://code.google.com/p/markdown4j/)
(New BSD License) Py4J (net.sf.py4j:py4j:0.9 - http://py4j.sourceforge.net/)
(New BSD License) Py4J (net.sf.py4j:py4j:0.10.1 - http://py4j.sourceforge.net/) - https://github.com/bartdag/py4j/blob/0.10.1/LICENSE.txt
(New BSD License) Py4J (net.sf.py4j:py4j:0.10.3 - http://py4j.sourceforge.net/) - https://github.com/bartdag/py4j/blob/0.10.3/LICENSE.txt
(New BSD License) Markdown4j (org.commonjava.googlecode.markdown4j:markdown4j:jar:2.2-cj-1.0 - https://code.google.com/p/markdown4j/)
(BSD 3 Clause) Paranamer (com.thoughtworks.paranamer:paranamer:jar:2.6) - https://github.com/paul-hammant/paranamer/blob/paranamer-parent-2.6/LICENSE.txt
(BSD 3 Clause) netlib core (com.github.fommil.netlib:core:1.1.2 - https://github.com/fommil/netlib-java/core)

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -33,7 +33,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
* and InterpreterGroup will have reference to these all interpreters.
*
* Remember, list of interpreters are dedicated to a note.
* (when InterpreterOption.perNoteSession==true)
* (when InterpreterOption.session==true)
* So InterpreterGroup internally manages map of [noteId, list of interpreters]
*
* A InterpreterGroup runs on interpreter process.
@ -203,6 +203,14 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
LOGGER.info("Destroy interpreter group " + getId() + " for note " + noteId);
List<Interpreter> intpForNote = this.get(noteId);
destroy(intpForNote);
if (remoteInterpreterProcess != null) {
remoteInterpreterProcess.dereference();
if (remoteInterpreterProcess.referenceCount() <= 0) {
remoteInterpreterProcess = null;
allInterpreterGroups.remove(id);
}
}
}
@ -222,6 +230,7 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -298,6 +298,7 @@ public class RemoteInterpreter extends Interpreter {
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;

21
zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java
View file

@ -39,7 +39,6 @@ import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.utils.InterpreterBindingUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
@ -162,7 +161,7 @@ public class NotebookRestApi {
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.persist(subject);
notebookServer.broadcastNote(note);
notebookServer.broadcastNoteList(subject);
notebookServer.broadcastNoteList(subject, userAndRoles);
return new JsonResponse<>(Status.OK).build();
}
@ -177,7 +176,7 @@ public class NotebookRestApi {
public Response bind(@PathParam("noteId") String noteId, String req) throws IOException {
List<String> settingIdList = gson.fromJson(req, new TypeToken<List<String>>() {
}.getType());
notebook.bindInterpretersToNote(noteId, settingIdList);
notebook.bindInterpretersToNote(SecurityUtils.getPrincipal(), noteId, settingIdList);
return new JsonResponse<>(Status.OK).build();
}
@ -199,7 +198,8 @@ public class NotebookRestApi {
@ZeppelinApi
public Response getNotebookList() throws IOException {
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
List<Map<String, String>> notesInfo = notebookServer.generateNotebooksInfo(false, subject);
List<Map<String, String>> notesInfo = notebookServer.generateNotebooksInfo(false, subject,
SecurityUtils.getRoles());
return new JsonResponse<>(Status.OK, "", notesInfo).build();
}
@ -278,7 +278,7 @@ public class NotebookRestApi {
note.setName(noteName);
note.persist(subject);
notebookServer.broadcastNote(note);
notebookServer.broadcastNoteList(subject);
notebookServer.broadcastNoteList(subject, SecurityUtils.getRoles());
return new JsonResponse<>(Status.CREATED, "", note.getId()).build();
}
@ -302,7 +302,7 @@ public class NotebookRestApi {
}
}
notebookServer.broadcastNoteList(subject);
notebookServer.broadcastNoteList(subject, SecurityUtils.getRoles());
return new JsonResponse<>(Status.OK, "").build();
}
@ -327,7 +327,7 @@ public class NotebookRestApi {
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
Note newNote = notebook.cloneNote(notebookId, newNoteName, subject);
notebookServer.broadcastNote(newNote);
notebookServer.broadcastNoteList(subject);
notebookServer.broadcastNoteList(subject, SecurityUtils.getRoles());
return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build();
}
@ -458,7 +458,7 @@ public class NotebookRestApi {
}
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
note.removeParagraph(paragraphId);
note.removeParagraph(SecurityUtils.getPrincipal(), paragraphId);
note.persist(subject);
notebookServer.broadcastNote(note);
@ -599,6 +599,11 @@ public class NotebookRestApi {
// handle params if presented
handleParagraphParams(message, note, paragraph);
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
paragraph.setAuthenticationInfo(subject);
note.persist(subject);
note.run(paragraph.getId());
return new JsonResponse<>(Status.OK).build();
}

2
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -89,7 +89,7 @@ public class ZeppelinServer extends Application {
this.heliumApplicationFactory = new HeliumApplicationFactory();
this.schedulerFactory = new SchedulerFactory();
this.replFactory = new InterpreterFactory(conf, notebookWsServer,
notebookWsServer, heliumApplicationFactory, depResolver);
notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated());
this.notebookRepo = new NotebookRepoSync(conf);
this.notebookIndex = new LuceneSearch();
this.notebookAuthorization = NotebookAuthorization.init(conf);

106
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -20,7 +20,6 @@ import com.google.common.base.Strings;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@ -134,7 +133,7 @@ public class NotebookServer extends WebSocketServlet implements
if (LOG.isTraceEnabled()) {
LOG.trace("RECEIVE MSG = " + messagereceived);
}
String ticket = TicketContainer.instance.getTicket(messagereceived.principal);
if (ticket != null && !ticket.equals(messagereceived.ticket)){
/* not to pollute logs, log instead of exception */
@ -172,10 +171,10 @@ public class NotebookServer extends WebSocketServlet implements
/** Lets be elegant here */
switch (messagereceived.op) {
case LIST_NOTES:
unicastNoteList(conn, subject);
unicastNoteList(conn, subject, userAndRoles);
break;
case RELOAD_NOTES_FROM_REPO:
broadcastReloadedNoteList(subject);
broadcastReloadedNoteList(subject, userAndRoles);
break;
case GET_HOME_NOTE:
sendHomeNote(conn, userAndRoles, notebook, messagereceived);
@ -472,7 +471,8 @@ public class NotebookServer extends WebSocketServlet implements
List<String> settingIdList = gson.fromJson(String.valueOf(
fromMessage.data.get("selectedSettingIds")), new TypeToken<ArrayList<String>>() {
}.getType());
notebook().bindInterpretersToNote(noteId, settingIdList);
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
notebook().bindInterpretersToNote(subject.getUser(), noteId, settingIdList);
broadcastInterpreterBindings(noteId,
InterpreterBindingUtils.getInterpreterBindings(notebook(), noteId));
} catch (Exception e) {
@ -490,7 +490,7 @@ public class NotebookServer extends WebSocketServlet implements
}
public List<Map<String, String>> generateNotebooksInfo(boolean needsReload,
AuthenticationInfo subject) {
AuthenticationInfo subject, HashSet<String> userAndRoles) {
Notebook notebook = notebook();
@ -507,7 +507,7 @@ public class NotebookServer extends WebSocketServlet implements
}
}
List<Note> notes = notebook.getAllNotes(subject);
List<Note> notes = notebook.getAllNotes(userAndRoles);
List<Map<String, String>> notesInfo = new LinkedList<>();
for (Note note : notes) {
Map<String, String> info = new HashMap<>();
@ -534,34 +534,35 @@ public class NotebookServer extends WebSocketServlet implements
.put("interpreterBindings", settingList));
}
public void broadcastNoteList(AuthenticationInfo subject) {
public void broadcastNoteList(AuthenticationInfo subject, HashSet userAndRoles) {
if (subject == null) {
subject = new AuthenticationInfo(StringUtils.EMPTY);
}
//send first to requesting user
List<Map<String, String>> notesInfo = generateNotebooksInfo(false, subject);
List<Map<String, String>> notesInfo = generateNotebooksInfo(false, subject, userAndRoles);
multicastToUser(subject.getUser(), new Message(OP.NOTES_INFO).put("notes", notesInfo));
//to others afterwards
for (String user: userConnectedSockets.keySet()) {
if (subject.getUser() == user) {
continue;
}
notesInfo = generateNotebooksInfo(false, new AuthenticationInfo(user));
notesInfo = generateNotebooksInfo(false, new AuthenticationInfo(user), userAndRoles);
multicastToUser(user, new Message(OP.NOTES_INFO).put("notes", notesInfo));
}
}
public void unicastNoteList(NotebookSocket conn, AuthenticationInfo subject) {
List<Map<String, String>> notesInfo = generateNotebooksInfo(false, subject);
public void unicastNoteList(NotebookSocket conn, AuthenticationInfo subject,
HashSet<String> userAndRoles) {
List<Map<String, String>> notesInfo = generateNotebooksInfo(false, subject, userAndRoles);
unicast(new Message(OP.NOTES_INFO).put("notes", notesInfo), conn);
}
public void broadcastReloadedNoteList(AuthenticationInfo subject) {
public void broadcastReloadedNoteList(AuthenticationInfo subject, HashSet userAndRoles) {
if (subject == null) {
subject = new AuthenticationInfo(StringUtils.EMPTY);
}
//reload and reply first to requesting user
List<Map<String, String>> notesInfo = generateNotebooksInfo(true, subject);
List<Map<String, String>> notesInfo = generateNotebooksInfo(true, subject, userAndRoles);
multicastToUser(subject.getUser(), new Message(OP.NOTES_INFO).put("notes", notesInfo));
//to others afterwards
for (String user: userConnectedSockets.keySet()) {
@ -569,7 +570,7 @@ public class NotebookServer extends WebSocketServlet implements
continue;
}
//reloaded already above; parameter - false
notesInfo = generateNotebooksInfo(false, new AuthenticationInfo(user));
notesInfo = generateNotebooksInfo(false, new AuthenticationInfo(user), userAndRoles);
multicastToUser(user, new Message(OP.NOTES_INFO).put("notes", notesInfo));
}
}
@ -600,6 +601,8 @@ public class NotebookServer extends WebSocketServlet implements
return;
}
String user = fromMessage.principal;
Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (note != null) {
@ -610,7 +613,7 @@ public class NotebookServer extends WebSocketServlet implements
}
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
sendAllAngularObjects(note, user, conn);
} else {
conn.send(serializeMessage(new Message(OP.NOTE).put("note", null)));
}
@ -619,6 +622,7 @@ public class NotebookServer extends WebSocketServlet implements
private void sendHomeNote(NotebookSocket conn, HashSet<String> userAndRoles,
Notebook notebook, Message fromMessage) throws IOException {
String noteId = notebook.getConf().getString(ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN);
String user = fromMessage.principal;
Note note = null;
if (noteId != null) {
@ -634,7 +638,7 @@ public class NotebookServer extends WebSocketServlet implements
}
addConnectionToNote(note.getId(), conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", note)));
sendAllAngularObjects(note, conn);
sendAllAngularObjects(note, user, conn);
} else {
removeConnectionFromAllNote(conn);
conn.send(serializeMessage(new Message(OP.NOTE).put("note", null)));
@ -674,7 +678,7 @@ public class NotebookServer extends WebSocketServlet implements
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
note.persist(subject);
broadcastNote(note);
broadcastNoteList(subject);
broadcastNoteList(subject, userAndRoles);
}
}
@ -709,7 +713,7 @@ public class NotebookServer extends WebSocketServlet implements
note.persist(subject);
addConnectionToNote(note.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
broadcastNoteList(subject);
broadcastNoteList(subject, userAndRoles);
}
private void removeNote(NotebookSocket conn, HashSet<String> userAndRoles,
@ -731,7 +735,7 @@ public class NotebookServer extends WebSocketServlet implements
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
notebook.removeNote(noteId, subject);
removeNote(noteId);
broadcastNoteList(subject);
broadcastNoteList(subject, userAndRoles);
}
private void updateParagraph(NotebookSocket conn, HashSet<String> userAndRoles,
@ -773,7 +777,7 @@ public class NotebookServer extends WebSocketServlet implements
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
addConnectionToNote(newNote.getId(), (NotebookSocket) conn);
conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", newNote)));
broadcastNoteList(subject);
broadcastNoteList(subject, userAndRoles);
}
protected Note importNote(NotebookSocket conn, HashSet<String> userAndRoles,
@ -786,11 +790,13 @@ public class NotebookServer extends WebSocketServlet implements
AuthenticationInfo subject = null;
if (fromMessage.principal != null) {
subject = new AuthenticationInfo(fromMessage.principal);
} else {
subject = new AuthenticationInfo("anonymous");
}
note = notebook.importNote(noteJson, noteName, subject);
note.persist(subject);
broadcastNote(note);
broadcastNoteList(subject);
broadcastNoteList(subject, userAndRoles);
}
return note;
}
@ -804,7 +810,7 @@ public class NotebookServer extends WebSocketServlet implements
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
@ -813,7 +819,7 @@ public class NotebookServer extends WebSocketServlet implements
/** We dont want to remove the last paragraph */
if (!note.isLastParagraph(paragraphId)) {
note.removeParagraph(paragraphId);
note.removeParagraph(subject.getUser(), paragraphId);
note.persist(subject);
broadcastNote(note);
}
@ -869,6 +875,7 @@ public class NotebookServer extends WebSocketServlet implements
String interpreterGroupId = (String) fromMessage.get("interpreterGroupId");
String varName = (String) fromMessage.get("name");
Object varValue = fromMessage.get("value");
String user = fromMessage.principal;
AngularObject ao = null;
boolean global = false;
// propagate change to (Remote) AngularObjectRegistry
@ -877,12 +884,12 @@ public class NotebookServer extends WebSocketServlet implements
List<InterpreterSetting> settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(note.getId()) == null) {
if (setting.getInterpreterGroup(user, note.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(note.getId()).getId())) {
if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
.getInterpreterGroup(note.getId()).getAngularObjectRegistry();
.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@ -919,12 +926,12 @@ public class NotebookServer extends WebSocketServlet implements
List<InterpreterSetting> settings = notebook.getInterpreterFactory()
.getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(n.getId()) == null) {
if (setting.getInterpreterGroup(user, n.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(n.getId()).getId())) {
if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
AngularObjectRegistry angularObjectRegistry = setting
.getInterpreterGroup(n.getId()).getAngularObjectRegistry();
.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
this.broadcastExcept(
n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
@ -1110,7 +1117,7 @@ public class NotebookServer extends WebSocketServlet implements
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
@ -1129,7 +1136,7 @@ public class NotebookServer extends WebSocketServlet implements
String noteId = getOpenNoteId(conn);
final Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "write", fromMessage.principal,
userAndRoles, notebookAuthorization.getWriters(noteId));
@ -1181,14 +1188,9 @@ public class NotebookServer extends WebSocketServlet implements
String text = (String) fromMessage.get("paragraph");
p.setText(text);
p.setTitle((String) fromMessage.get("title"));
if (!fromMessage.principal.equals("anonymous")) {
AuthenticationInfo authenticationInfo = new AuthenticationInfo(fromMessage.principal,
fromMessage.ticket);
p.setAuthenticationInfo(authenticationInfo);
} else {
p.setAuthenticationInfo(new AuthenticationInfo());
}
AuthenticationInfo authenticationInfo =
new AuthenticationInfo(fromMessage.principal, fromMessage.ticket);
p.setAuthenticationInfo(authenticationInfo);
Map<String, Object> params = (Map<String, Object>) fromMessage
.get("params");
@ -1513,7 +1515,7 @@ public class NotebookServer extends WebSocketServlet implements
LOG.info("Job {} is finished", job.getId());
try {
//TODO(khalid): may change interface for JobListener and pass subject from interpreter
note.persist(null);
note.persist(job instanceof Paragraph ? ((Paragraph) job).getAuthenticationInfo() : null);
} catch (IOException e) {
LOG.error(e.toString(), e);
}
@ -1569,7 +1571,8 @@ public class NotebookServer extends WebSocketServlet implements
return new NotebookInformationListener(this);
}
private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOException {
private void sendAllAngularObjects(Note note, String user, NotebookSocket conn)
throws IOException {
List<InterpreterSetting> settings =
notebook().getInterpreterFactory().getInterpreterSettings(note.getId());
if (settings == null || settings.size() == 0) {
@ -1577,17 +1580,15 @@ public class NotebookServer extends WebSocketServlet implements
}
for (InterpreterSetting intpSetting : settings) {
AngularObjectRegistry registry = intpSetting.getInterpreterGroup(note.getId())
.getAngularObjectRegistry();
AngularObjectRegistry registry =
intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
conn.send(serializeMessage(new Message(OP.ANGULAR_OBJECT_UPDATE)
.put("angularObject", object)
.put("interpreterGroupId",
intpSetting.getInterpreterGroup(note.getId()).getId())
.put("noteId", note.getId())
.put("paragraphId", object.getParagraphId())
));
conn.send(serializeMessage(
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
.put("interpreterGroupId",
intpSetting.getInterpreterGroup(user, note.getId()).getId())
.put("noteId", note.getId()).put("paragraphId", object.getParagraphId())));
}
}
}
@ -1654,9 +1655,10 @@ public class NotebookServer extends WebSocketServlet implements
String paragraphId = (String) fromMessage.get("paragraphId");
String replName = (String) fromMessage.get("magic");
String noteId = getOpenNoteId(conn);
String user = fromMessage.principal;
Message resp = new Message(OP.EDITOR_SETTING);
resp.put("paragraphId", paragraphId);
resp.put("editor", notebook().getInterpreterFactory().getEditorSetting(noteId, replName));
resp.put("editor", notebook().getInterpreterFactory().getEditorSetting(user, noteId, replName));
conn.send(serializeMessage(resp));
return;
}

6
zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java
View file

@ -119,4 +119,10 @@ public class SecurityUtils {
return roles;
}
/**
* Checked if shiro enabled or not
*/
public static boolean isAuthenticated() {
return org.apache.shiro.SecurityUtils.getSubject().isAuthenticated();
}
}

25
zeppelin-server/src/main/resources/log4j.properties Normal file
View file

@ -0,0 +1,25 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
log4j.rootLogger = INFO, stdout
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L) - %m%n
log4j.additivity.org.apache.zeppelin.interpreter = false
log4j.logger.org.apache.zeppelin.interpreter = DEBUG, stdout

11
zeppelin-server/src/main/resources/shiro.ini
View file

@ -18,14 +18,19 @@
[users]
# List of users with their password allowed to access Zeppelin.
# To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections
admin = password
admin = password, admin
user1 = user1, role1
[urls]
# anon means the access is anonymous.
# authcBasic means Basic Auth Security
# To enfore security, comment the line below and uncomment the next one
/** = anon
#/** = authcBasic
#/** = authc
[roles]
role1 = *
role2 = *
role3 = *
admin = *

2
zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
View file

@ -202,7 +202,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
try {
WebElement element = pollingWait(By.xpath("//*[@id='notebook-names']//a[contains(@href, '" + noteId + "')]"),
MAX_BROWSER_TIMEOUT_SEC);
collector.checkThat("Check is user has permission to view this notebook link", false,
collector.checkThat("Check is user has permission to view this notebook link", true,
CoreMatchers.equalTo(element.isDisplayed()));
} catch (Exception e) {
//This should have failed, nothing to worry.

31
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -31,7 +31,9 @@ import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@ -48,6 +50,7 @@ import static org.junit.Assert.*;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class InterpreterRestApiTest extends AbstractTestRestApi {
Gson gson = new Gson();
AuthenticationInfo anonymous;
@BeforeClass
public static void init() throws Exception {
@ -59,6 +62,11 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
AbstractTestRestApi.shutDown();
}
@Before
public void setUp() {
anonymous = new AuthenticationInfo("anonymous");
}
@Test
public void getAvailableInterpreters() throws IOException {
// when
@ -91,7 +99,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
"\"option\": { \"remote\": true, \"perNoteSession\": false }}";
"\"option\": { \"remote\": true, \"session\": false }}";
PostMethod post = httpPost("/interpreter/setting/", jsonRequest);
LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
assertThat("test create method:", post, isCreated());
@ -107,7 +115,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
"\"option\": { \"remote\": true, \"perNoteSession\": false }}";
"\"option\": { \"remote\": true, \"session\": false }}";
PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest);
LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString());
assertThat("test update method:", put, isAllowed());
@ -132,7 +140,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testInterpreterAutoBinding() throws IOException {
// create note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
// check interpreter is binded
GetMethod get = httpGet("/notebook/interpreter/bind/" + note.getId());
@ -145,13 +153,13 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testInterpreterRestart() throws IOException, InterruptedException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.addParagraph();
Paragraph p = note.getLastParagraph();
Map config = p.getConfig();
@ -160,6 +168,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
// run markdown paragraph
p.setConfig(config);
p.setText("%md markdown");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
@ -182,13 +191,14 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
p = note.addParagraph();
p.setConfig(config);
p.setText("%md markdown restarted");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
while (p.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
//cleanup
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
@ -221,22 +231,19 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
String jsonRequest = "{\"noteId\":\"" + note.getId() + "\"}";
// Restart isolated mode of Interpreter for note.
mdIntpSetting.getOption().setPerNoteProcess(true);
mdIntpSetting.getOption().setPerNoteSession(false);
mdIntpSetting.getOption().setPerNote(InterpreterOption.ISOLATED);
PutMethod put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("isolated interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart scoped mode of Interpreter for note.
mdIntpSetting.getOption().setPerNoteSession(true);
mdIntpSetting.getOption().setPerNoteProcess(false);
mdIntpSetting.getOption().setPerNote(InterpreterOption.SCOPED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("scoped interpreter restart:", put, isAllowed());
put.releaseConnection();
// Restart shared mode of Interpreter for note.
mdIntpSetting.getOption().setPerNoteProcess(false);
mdIntpSetting.getOption().setPerNoteSession(false);
mdIntpSetting.getOption().setPerNote(InterpreterOption.SHARED);
put = httpPut("/interpreter/setting/restart/" + mdIntpSetting.getId(), jsonRequest);
assertThat("shared interpreter restart:", put, isAllowed());
put.releaseConnection();

26
zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
View file

@ -28,7 +28,9 @@ import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.NotebookAuthorizationInfoSaving;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@ -49,6 +51,7 @@ import static org.junit.Assert.assertThat;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class NotebookRestApiTest extends AbstractTestRestApi {
Gson gson = new Gson();
AuthenticationInfo anonymous;
@BeforeClass
public static void init() throws Exception {
@ -60,9 +63,14 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
AbstractTestRestApi.shutDown();
}
@Before
public void setUp() {
anonymous = new AuthenticationInfo("anonymous");
}
@Test
public void testPermissions() throws IOException {
Note note1 = ZeppelinServer.notebook.createNote(null);
Note note1 = ZeppelinServer.notebook.createNote(anonymous);
// Set only readers
String jsonRequest = "{\"readers\":[\"admin-team\"],\"owners\":[]," +
"\"writers\":[]}";
@ -85,7 +93,7 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
get.releaseConnection();
Note note2 = ZeppelinServer.notebook.createNote(null);
Note note2 = ZeppelinServer.notebook.createNote(anonymous);
// Set only writers
jsonRequest = "{\"readers\":[],\"owners\":[]," +
"\"writers\":[\"admin-team\"]}";
@ -119,14 +127,14 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
assertEquals(authInfo.get("owners"), Lists.newArrayList());
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(note2.getId(), null);
ZeppelinServer.notebook.removeNote(note1.getId(), anonymous);
ZeppelinServer.notebook.removeNote(note2.getId(), anonymous);
}
@Test
public void testGetNoteParagraphJobStatus() throws IOException {
Note note1 = ZeppelinServer.notebook.createNote(null);
Note note1 = ZeppelinServer.notebook.createNote(anonymous);
note1.addParagraph();
String paragraphId = note1.getLastParagraph().getId();
@ -142,13 +150,13 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
assertEquals(paragraphStatus.get("status"), "READY");
//cleanup
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(note1.getId(), anonymous);
}
@Test
public void testCloneNotebook() throws IOException {
Note note1 = ZeppelinServer.notebook.createNote(null);
Note note1 = ZeppelinServer.notebook.createNote(anonymous);
PostMethod post = httpPost("/notebook/" + note1.getId(), "");
LOG.info("testCloneNotebook response\n" + post.getResponseBodyAsString());
assertThat(post, isCreated());
@ -167,8 +175,8 @@ public class NotebookRestApiTest extends AbstractTestRestApi {
get.releaseConnection();
//cleanup
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(clonedNotebookId, null);
ZeppelinServer.notebook.removeNote(note1.getId(), anonymous);
ZeppelinServer.notebook.removeNote(clonedNotebookId, anonymous);
}
}

112
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java
View file

@ -19,9 +19,11 @@ package org.apache.zeppelin.rest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import com.google.common.collect.Sets;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
@ -34,6 +36,7 @@ import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.FixMethodOrder;
import org.junit.Test;
@ -51,6 +54,7 @@ import static org.junit.Assert.*;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class ZeppelinRestApiTest extends AbstractTestRestApi {
Gson gson = new Gson();
AuthenticationInfo anonymous;
@BeforeClass
public static void init() throws Exception {
@ -62,6 +66,11 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
AbstractTestRestApi.shutDown();
}
@Before
public void setUp() {
anonymous = new AuthenticationInfo("anonymous");
}
/***
* ROOT API TEST
***/
@ -78,7 +87,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
public void testGetNotebookInfo() throws IOException {
LOG.info("testGetNotebookInfo");
// Create note to get info
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("note");
Paragraph paragraph = note.addParagraph();
@ -87,7 +96,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
paragraph.setConfig(config);
String paragraphText = "%md This is my new paragraph in my new note";
paragraph.setText(paragraphText);
note.persist(null);
note.persist(anonymous);
String sourceNoteID = note.getId();
GetMethod get = httpGet("/notebook/" + sourceNoteID);
@ -106,7 +115,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertTrue(paragraphs.size() > 0);
assertEquals(paragraphText, paragraphs.get(0).get("text"));
//
ZeppelinServer.notebook.removeNote(sourceNoteID, null);
ZeppelinServer.notebook.removeNote(sourceNoteID, anonymous);
}
@Test
@ -156,7 +165,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertTrue("paragraph text check failed", p.getText().startsWith("text"));
}
// cleanup
ZeppelinServer.notebook.removeNote(newNotebookId, null);
ZeppelinServer.notebook.removeNote(newNotebookId, anonymous);
post.releaseConnection();
}
@ -183,7 +192,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
}
assertEquals("compare note name", expectedNoteName, newNoteName);
// cleanup
ZeppelinServer.notebook.removeNote(newNotebookId, null);
ZeppelinServer.notebook.removeNote(newNotebookId, anonymous);
post.releaseConnection();
}
@ -192,7 +201,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
public void testDeleteNote() throws IOException {
LOG.info("testDeleteNote");
//Create note and get ID
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
String noteId = note.getId();
testDeleteNotebook(noteId);
}
@ -208,7 +217,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
@Test
public void testExportNotebook() throws IOException {
LOG.info("testExportNotebook");
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("source note for export");
Paragraph paragraph = note.addParagraph();
@ -216,7 +225,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
note.persist(null);
note.persist(anonymous);
String sourceNoteID = note.getId();
// Call export Notebook REST API
GetMethod get = httpGet("/notebook/export/" + sourceNoteID);
@ -230,7 +239,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
String exportJSON = (String) resp.get("body");
assertNotNull("Can not find new notejson", exportJSON);
LOG.info("export JSON:=" + exportJSON);
ZeppelinServer.notebook.removeNote(sourceNoteID, null);
ZeppelinServer.notebook.removeNote(sourceNoteID, anonymous);
get.releaseConnection();
}
@ -241,7 +250,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
String noteName = "source note for import";
LOG.info("testImortNotebook");
// create test notebook
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName(noteName);
Paragraph paragraph = note.addParagraph();
@ -249,7 +258,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
note.persist(null);
note.persist(anonymous);
String sourceNoteID = note.getId();
// get note content as JSON
String oldJson = getNoteContent(sourceNoteID);
@ -267,8 +276,8 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("Compare paragraphs count", note.getParagraphs().size(), newNote.getParagraphs()
.size());
// cleanup
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(newNote.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
ZeppelinServer.notebook.removeNote(newNote.getId(), anonymous);
importPost.releaseConnection();
}
@ -303,7 +312,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
public void testCloneNotebook() throws IOException, CloneNotSupportedException, IllegalArgumentException {
LOG.info("testCloneNotebook");
// Create note to clone
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("source note for clone");
Paragraph paragraph = note.addParagraph();
@ -311,7 +320,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
config.put("enabled", true);
paragraph.setConfig(config);
paragraph.setText("%md This is my new paragraph in my new note");
note.persist(null);
note.persist(anonymous);
String sourceNoteID = note.getId();
String noteName = "clone Note Name";
@ -331,8 +340,8 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("Compare note names", noteName, newNote.getName());
assertEquals("Compare paragraphs count", note.getParagraphs().size(), newNote.getParagraphs().size());
//cleanup
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(newNote.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
ZeppelinServer.notebook.removeNote(newNote.getId(), anonymous);
post.releaseConnection();
}
@ -345,8 +354,8 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
}.getType());
List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body");
//TODO(khalid): anonymous or specific user notes?
AuthenticationInfo subject = new AuthenticationInfo("anonymous");
assertEquals("List notebooks are equal", ZeppelinServer.notebook.getAllNotes(subject).size(), body.size());
HashSet<String> anonymous = Sets.newHashSet("anonymous");
assertEquals("List notebooks are equal", ZeppelinServer.notebook.getAllNotes(anonymous).size(), body.size());
get.releaseConnection();
}
@ -354,7 +363,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
public void testNoteJobs() throws IOException, InterruptedException {
LOG.info("testNoteJobs");
// Create note to run test.
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@ -364,7 +373,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
paragraph.setConfig(config);
paragraph.setText("%md This is test paragraph.");
note.persist(null);
note.persist(anonymous);
String noteID = note.getId();
note.runAll();
@ -402,14 +411,14 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
Thread.sleep(1000);
//cleanup
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testGetNotebookJob() throws IOException, InterruptedException {
LOG.info("testGetNotebookJob");
// Create note to run test.
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@ -419,7 +428,8 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
paragraph.setConfig(config);
paragraph.setText("%sh sleep 1");
note.persist(null);
paragraph.setAuthenticationInfo(anonymous);
note.persist(anonymous);
String noteID = note.getId();
note.runAll();
@ -455,14 +465,14 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
}
}
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRunParagraphWithParams() throws IOException, InterruptedException {
LOG.info("testRunParagraphWithParams");
// Create note to run test.
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
assertNotNull("can't create new note", note);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@ -472,7 +482,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
paragraph.setConfig(config);
paragraph.setText("%spark\nval param = z.input(\"param\").toString\nprintln(param)");
note.persist(null);
note.persist(anonymous);
String noteID = note.getId();
note.runAll();
@ -500,13 +510,13 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("world", params.get("param2"));
//cleanup
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testCronJobs() throws InterruptedException, IOException{
public void testJobs() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
@ -550,18 +560,18 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
DeleteMethod deleteCron = httpDelete("/notebook/cron/" + note.getId());
assertThat("", deleteCron, isAllowed());
deleteCron.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRegressionZEPPELIN_527() throws IOException {
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.setName("note for run test");
Paragraph paragraph = note.addParagraph();
paragraph.setText("%spark\nval param = z.input(\"param\").toString\nprintln(param)");
note.persist(null);
note.persist(anonymous);
GetMethod getNoteJobs = httpGet("/notebook/job/" + note.getId());
assertThat("test notebook jobs run:", getNoteJobs, isAllowed());
@ -572,12 +582,12 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertFalse(body.get(0).containsKey("finished"));
getNoteJobs.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testInsertParagraph() throws IOException {
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
String jsonRequest = "{\"title\": \"title1\", \"text\": \"text1\"}";
PostMethod post = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest);
@ -612,17 +622,17 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("title2", paragraphAtIdx0.getTitle());
assertEquals("text2", paragraphAtIdx0.getText());
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testGetParagraph() throws IOException {
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
Paragraph p = note.addParagraph();
p.setTitle("hello");
p.setText("world");
note.persist(null);
note.persist(anonymous);
GetMethod get = httpGet("/notebook/" + note.getId() + "/paragraph/" + p.getId());
LOG.info("testGetParagraph response\n" + get.getResponseBodyAsString());
@ -641,12 +651,12 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertEquals("hello", body.get("title"));
assertEquals("world", body.get("text"));
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testMoveParagraph() throws IOException {
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
Paragraph p = note.addParagraph();
p.setTitle("title1");
@ -656,7 +666,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
p2.setTitle("title2");
p2.setText("text2");
note.persist(null);
note.persist(anonymous);
PostMethod post = httpPost("/notebook/" + note.getId() + "/paragraph/" + p2.getId() + "/move/" + 0, "");
assertThat("Test post method: ", post, isAllowed());
@ -673,18 +683,18 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
assertThat("Test post method: ", post2, isBadRequest());
post.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testDeleteParagraph() throws IOException {
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
Paragraph p = note.addParagraph();
p.setTitle("title1");
p.setText("text1");
note.persist(null);
note.persist(anonymous);
DeleteMethod delete = httpDelete("/notebook/" + note.getId() + "/paragraph/" + p.getId());
assertThat("Test delete method: ", delete, isAllowed());
@ -694,7 +704,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
Paragraph retrParagrah = retrNote.getParagraph(p.getId());
assertNull("paragraph should be deleted", retrParagrah);
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
@ -710,12 +720,12 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
String username = body.get("principal");
getSecurityTicket.releaseConnection();
Note note1 = ZeppelinServer.notebook.createNote(null);
Note note1 = ZeppelinServer.notebook.createNote(anonymous);
String jsonRequest = "{\"title\": \"title1\", \"text\": \"ThisIsToTestSearchMethodWithPermissions 1\"}";
PostMethod postNotebookText = httpPost("/notebook/" + note1.getId() + "/paragraph", jsonRequest);
postNotebookText.releaseConnection();
Note note2 = ZeppelinServer.notebook.createNote(null);
Note note2 = ZeppelinServer.notebook.createNote(anonymous);
jsonRequest = "{\"title\": \"title1\", \"text\": \"ThisIsToTestSearchMethodWithPermissions 2\"}";
postNotebookText = httpPost("/notebook/" + note2.getId() + "/paragraph", jsonRequest);
postNotebookText.releaseConnection();
@ -757,13 +767,13 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
getPermission.releaseConnection();
}
searchNotebook.releaseConnection();
ZeppelinServer.notebook.removeNote(note1.getId(), null);
ZeppelinServer.notebook.removeNote(note2.getId(), null);
ZeppelinServer.notebook.removeNote(note1.getId(), anonymous);
ZeppelinServer.notebook.removeNote(note2.getId(), anonymous);
}
@Test
public void testTitleSearch() throws IOException {
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
String jsonRequest = "{\"title\": \"testTitleSearchOfParagraph\", \"text\": \"ThisIsToTestSearchMethodWithTitle \"}";
PostMethod postNotebookText = httpPost("/notebook/" + note.getId() + "/paragraph", jsonRequest);
postNotebookText.releaseConnection();
@ -784,7 +794,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
}
assertEquals("Paragraph title hits must be at-least one", true, numberOfTitleHits >= 1);
searchNotebook.releaseConnection();
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
}

54
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
View file

@ -31,7 +31,9 @@ import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@ -43,6 +45,7 @@ import com.google.gson.Gson;
*/
public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
Gson gson = new Gson();
AuthenticationInfo anonymous;
@BeforeClass
public static void init() throws Exception {
@ -54,6 +57,11 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
AbstractTestRestApi.shutDown();
}
@Before
public void setUp() {
anonymous = new AuthenticationInfo("anonymous");
}
private void waitForFinish(Paragraph p) {
while (p.getStatus() != Status.FINISHED
&& p.getStatus() != Status.ERROR
@ -69,7 +77,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
@Test
public void basicRDDTransformationAndActionTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
// run markdown paragraph, again
Paragraph p = note.addParagraph();
@ -77,17 +85,18 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
config.put("enabled", true);
p.setConfig(config);
p.setText("%spark print(sc.parallelize(1 to 10).reduce(_ + _))");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("55", p.getResult().message());
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void sparkSQLTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
int sparkVersion = getSparkVersionNumber(note);
// DataFrame API is available from spark 1.3
if (sparkVersion >= 13) {
@ -98,6 +107,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
p.setConfig(config);
p.setText("%spark val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n" +
"df.collect()");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
@ -111,6 +121,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
p.setConfig(config);
p.setText("%spark val df=sqlContext.createDataFrame(Seq((\"hello\",20)))\n" +
"z.show(df)");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
@ -125,20 +136,21 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
p.setConfig(config);
p.setText("%spark val ds=spark.createDataset(Seq((\"hello\",20)))\n" +
"z.show(ds)");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
assertEquals(InterpreterResult.Type.TABLE, p.getResult().type());
assertEquals("_1\t_2\nhello\t20\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
}
@Test
public void sparkRTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
int sparkVersion = getSparkVersionNumber(note);
if (isSparkR() && sparkVersion >= 14) { // sparkr supported from 1.4.0
@ -165,19 +177,20 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
"df <- createDataFrame(" + sqlContextName + ", localDF)\n" +
"count(df)"
);
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
System.err.println("sparkRTest=" + p.getResult().message());
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("[1] 3", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void pySparkTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.setName("note");
int sparkVersion = getSparkVersionNumber(note);
@ -188,6 +201,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
config.put("enabled", true);
p.setConfig(config);
p.setText("%pyspark print(sc.parallelize(range(1, 11)).reduce(lambda a, b: a + b))");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
@ -201,6 +215,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
p.setText("%pyspark from pyspark.sql import Row\n" +
"df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" +
"df.collect()");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
@ -214,6 +229,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
p.setText("%pyspark from pyspark.sql import Row\n" +
"df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" +
"z.show(df)");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
@ -228,6 +244,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
p.setConfig(config);
p.setText("%pyspark sqlContext.udf.register(\"f1\", lambda x: len(x))\n" +
"sqlContext.sql(\"select f1(\\\"abc\\\") as len\").collect()");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
@ -242,6 +259,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
p.setText("%pyspark from pyspark.sql import Row\n" +
"df=sqlContext.createDataFrame([Row(id=1, age=20)])\n" +
"df.collect()");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
@ -255,19 +273,20 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
// use SQLContext to register UDF but use this UDF through SparkSession
p.setText("%pyspark sqlContext.udf.register(\"f1\", lambda x: len(x))\n" +
"spark.sql(\"select f1(\\\"abc\\\") as len\").collect()");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("[Row(len=u'3')]\n", p.getResult().message());
}
}
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void pySparkAutoConvertOptionTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
note.setName("note");
int sparkVersionNumber = getSparkVersionNumber(note);
@ -286,33 +305,37 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
p.setText("%pyspark\nfrom pyspark.sql.functions import *\n"
+ "print(" + sqlContextName + ".range(0, 10).withColumn('uniform', rand(seed=10) * 3.14).count())");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());
assertEquals("10\n", p.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void zRunTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
Paragraph p0 = note.addParagraph();
Map config0 = p0.getConfig();
config0.put("enabled", true);
p0.setConfig(config0);
p0.setText("%spark z.run(1)");
p0.setAuthenticationInfo(anonymous);
Paragraph p1 = note.addParagraph();
Map config1 = p1.getConfig();
config1.put("enabled", true);
p1.setConfig(config1);
p1.setText("%spark val a=10");
p1.setAuthenticationInfo(anonymous);
Paragraph p2 = note.addParagraph();
Map config2 = p2.getConfig();
config2.put("enabled", true);
p2.setConfig(config2);
p2.setText("%spark print(a)");
p2.setAuthenticationInfo(anonymous);
note.run(p0.getId());
waitForFinish(p0);
@ -323,13 +346,13 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
assertEquals(Status.FINISHED, p2.getStatus());
assertEquals("10", p2.getResult().message());
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
@Test
public void pySparkDepLoaderTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
Note note = ZeppelinServer.notebook.createNote(anonymous);
int sparkVersionNumber = getSparkVersionNumber(note);
if (isPyspark() && sparkVersionNumber >= 14) {
@ -350,6 +373,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
config.put("enabled", true);
p0.setConfig(config);
p0.setText("%dep z.load(\"com.databricks:spark-csv_2.11:1.2.0\")");
p0.setAuthenticationInfo(anonymous);
note.run(p0.getId());
waitForFinish(p0);
assertEquals(Status.FINISHED, p0.getStatus());
@ -370,13 +394,14 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
"from pyspark.sql import SQLContext\n" +
"print(" + sqlContextName + ".read.format('com.databricks.spark.csv')" +
".load('"+ tmpFile.getAbsolutePath() +"').count())");
p1.setAuthenticationInfo(anonymous);
note.run(p1.getId());
waitForFinish(p1);
assertEquals(Status.FINISHED, p1.getStatus());
assertEquals("2\n", p1.getResult().message());
}
ZeppelinServer.notebook.removeNote(note.getId(), null);
ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
}
/**
@ -390,6 +415,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
config.put("enabled", true);
p.setConfig(config);
p.setText("%spark print(sc.version)");
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
waitForFinish(p);
assertEquals(Status.FINISHED, p.getStatus());

12
zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
View file

@ -31,6 +31,7 @@ import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.apache.zeppelin.rest.AbstractTestRestApi;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@ -57,6 +58,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
private static NotebookServer notebookServer;
private static Gson gson;
private HttpServletRequest mockRequest;
private AuthenticationInfo anonymous;
@BeforeClass
public static void init() throws Exception {
@ -74,6 +76,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
@Before
public void setUp() {
mockRequest = mock(HttpServletRequest.class);
anonymous = new AuthenticationInfo("anonymous");
}
@Test
@ -94,14 +97,14 @@ public class NotebookServerTest extends AbstractTestRestApi {
@Test
public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException {
// create a notebook
Note note1 = notebook.createNote(null);
Note note1 = notebook.createNote(anonymous);
// get reference to interpreterGroup
InterpreterGroup interpreterGroup = null;
List<InterpreterSetting> settings = notebook.getInterpreterFactory().getInterpreterSettings(note1.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("md")) {
interpreterGroup = setting.getInterpreterGroup("sharedProcess");
interpreterGroup = setting.getInterpreterGroup("anonymous", "sharedProcess");
break;
}
}
@ -109,6 +112,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
// start interpreter process
Paragraph p1 = note1.addParagraph();
p1.setText("%md start remote interpreter process");
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
// add angularObject
@ -144,7 +148,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
verify(sock1, times(0)).send(anyString());
verify(sock2, times(1)).send(anyString());
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@Test
@ -167,7 +171,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
assertNotEquals(null, notebook.getNote(note.getId()));
assertEquals("Test Zeppelin notebook import", notebook.getNote(note.getId()).getName());
assertEquals("Test paragraphs import", notebook.getNote(note.getId()).getParagraphs().get(0).getText());
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test

15
zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html
View file

@ -37,23 +37,31 @@ limitations under the License.
<div>
<h5>Option</h5>
<span class="checkbox input-group">
<label><input type="checkbox" style="width:0%;height:0%" id="perNote" ng-model="newInterpreterSetting.option.perNote"/>
perNote </label>
</span>
<span class="checkbox input-group">
<label><input type="checkbox" style="width:0%;height:0%" id="perUser" ng-model="newInterpreterSetting.option.perUser"/>
perUser</label>
</span>
<span class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
data-toggle="dropdown">
{{getSessionOption(setting.id)}} <span class="caret"></span>
{{getSessionOption(newInterpreterSetting.id)}} <span class="caret"></span>
</button>
<ul class="dropdown-menu" role="menu">
<li>
<a style="cursor:pointer"
tooltip="Single interpreter instance are shared across notes"
ng-click="setSessionOption(setting.id, 'shared')">
ng-click="setSessionOption(newInterpreterSetting.id, 'shared')">
shared
</a>
</li>
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter instance for each note"
ng-click="setSessionOption(setting.id, 'scoped')">
ng-click="setSessionOption(newInterpreterSetting.id, 'scoped')">
scoped
</a>
</li>
@ -66,7 +74,6 @@ limitations under the License.
</li>
</ul>
</span>
<span>Interpreter for note</span>
</div>
<div class="row interpreter" style="margin-top: 5px;">

131
zeppelin-web/src/app/interpreter/interpreter.controller.js
View file

@ -16,9 +16,9 @@
angular.module('zeppelinWebApp').controller('InterpreterCtrl', InterpreterCtrl);
InterpreterCtrl.$inject = ['$scope', '$http', 'baseUrlSrv', 'ngToast', '$timeout', '$route'];
InterpreterCtrl.$inject = ['$rootScope', '$scope', '$http', 'baseUrlSrv', 'ngToast', '$timeout', '$route'];
function InterpreterCtrl($scope, $http, baseUrlSrv, ngToast, $timeout, $route) {
function InterpreterCtrl($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeout, $route) {
var interpreterSettingsTmp = [];
$scope.interpreterSettings = [];
$scope.availableInterpreters = {};
@ -156,7 +156,7 @@
interpreterSettingsTmp[index] = angular.copy($scope.interpreterSettings[index]);
};
$scope.setSessionOption = function(settingId, sessionOption) {
$scope.setPerNoteOption = function(settingId, sessionOption) {
var option;
if (settingId === undefined) {
option = $scope.newInterpreterSetting.option;
@ -167,18 +167,21 @@
}
if (sessionOption === 'isolated') {
option.perNoteSession = false;
option.perNoteProcess = true;
option.perNote = sessionOption;
option.session = false;
option.process = true;
} else if (sessionOption === 'scoped') {
option.perNoteSession = true;
option.perNoteProcess = false;
option.perNote = sessionOption;
option.session = true;
option.process = false;
} else {
option.perNoteSession = false;
option.perNoteProcess = false;
option.perNote = 'shared';
option.session = false;
option.process = false;
}
};
$scope.getSessionOption = function(settingId) {
$scope.setPerUserOption = function(settingId, sessionOption) {
var option;
if (settingId === undefined) {
option = $scope.newInterpreterSetting.option;
@ -187,15 +190,115 @@
var setting = $scope.interpreterSettings[index];
option = setting.option;
}
if (option.perNoteSession) {
if (sessionOption === 'isolated') {
option.perUser = sessionOption;
option.session = false;
option.process = true;
} else if (sessionOption === 'scoped') {
option.perUser = sessionOption;
option.session = true;
option.process = false;
} else {
option.perUser = 'shared';
option.session = false;
option.process = false;
}
};
$scope.getPerNoteOption = function(settingId) {
var option;
if (settingId === undefined) {
option = $scope.newInterpreterSetting.option;
} else {
var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
var setting = $scope.interpreterSettings[index];
option = setting.option;
}
if (option.perNote === 'scoped') {
return 'scoped';
} else if (option.perNoteProcess) {
} else if (option.perNote === 'isolated') {
return 'isolated';
} else {
return 'shared';
}
};
$scope.getPerUserOption = function(settingId) {
var option;
if (settingId === undefined) {
option = $scope.newInterpreterSetting.option;
} else {
var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
var setting = $scope.interpreterSettings[index];
option = setting.option;
}
if (option.perUser === 'scoped') {
return 'scoped';
} else if (option.perUser === 'isolated') {
return 'isolated';
} else {
return 'shared';
}
};
$scope.getInterpreterRunningOption = function(settingId) {
var sharedModeName = 'shared';
var globallyModeName = 'Globally';
var perNoteModeName = 'Per Note';
var perUserModeName = 'Per User';
var option;
if (settingId === undefined) {
option = $scope.newInterpreterSetting.option;
} else {
var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
var setting = $scope.interpreterSettings[index];
option = setting.option;
}
var perNote = option.perNote;
var perUser = option.perUser;
// Globally == shared_perNote + shared_perUser
if (perNote === sharedModeName && perUser === sharedModeName) {
return globallyModeName;
}
if ($rootScope.ticket.ticket === 'anonymous' && $rootScope.ticket.roles === '[]') {
if (perNote !== undefined && typeof perNote === 'string' && perNote !== '') {
return perNoteModeName;
}
} else if ($rootScope.ticket.ticket !== 'anonymous') {
if (perNote !== undefined && typeof perNote === 'string' && perNote !== '') {
if (perUser !== undefined && typeof perUser === 'string' && perUser !== '') {
return perUserModeName;
}
return perNoteModeName;
}
}
option.perNote = sharedModeName;
option.perUser = sharedModeName;
return globallyModeName;
};
$scope.setInterpreterRunningOption = function(settingId, isPerNoteMode, isPerUserMode) {
var option;
if (settingId === undefined) {
option = $scope.newInterpreterSetting.option;
} else {
var index = _.findIndex($scope.interpreterSettings, {'id': settingId});
var setting = $scope.interpreterSettings[index];
option = setting.option;
}
option.perNote = isPerNoteMode;
option.perUser = isPerUserMode;
};
$scope.updateInterpreterSetting = function(form, settingId) {
var thisConfirm = BootstrapDialog.confirm({
closable: false,
@ -402,8 +505,8 @@
remote: true,
isExistingProcess: false,
setPermission: false,
perNoteSession: false,
perNoteProcess: false
session: false,
process: false
}
};

194
zeppelin-web/src/app/interpreter/interpreter.html
View file

@ -141,40 +141,172 @@ limitations under the License.
</span>
</div>
</div>
<div class="row interpreter">
<div class="col-md-12">
<h5>Option</h5>
<span class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
data-toggle="dropdown"
ng-disabled="!valueform.$visible">
{{getSessionOption(setting.id)}} <span class="caret"></span>
</button>
<ul class="dropdown-menu" role="menu">
<li>
<a style="cursor:pointer"
tooltip="Single interpreter instance are shared across notes"
ng-click="setSessionOption(setting.id, 'shared')">
shared
</a>
</li>
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter instance for each note"
ng-click="setSessionOption(setting.id, 'scoped')">
scoped
</a>
</li>
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter process for each note"
ng-click="setSessionOption(setting.id, 'isolated')">
isolated
</a>
</li>
</ul>
</span>
<span>Interpreter for note</span>
<div class="row interpreter" style="margin-top: 5px;">
<div class="col-md-6">
The interpreter will be instantiated
<span class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
data-toggle="dropdown"
ng-disabled="!valueform.$visible">
{{getInterpreterRunningOption(setting.id)}} <span class="caret"></span>
</button>
<ul class="dropdown-menu" role="menu">
<li>
<a style="cursor:pointer"
ng-click="setInterpreterRunningOption(setting.id, 'shared', 'shared')">
Globally
</a>
</li>
<li>
<a style="cursor:pointer"
ng-click="setInterpreterRunningOption(setting.id, 'scoped', '')">
Per Note
</a>
</li>
<li ng-if="ticket.ticket !== 'anonymous' && ticket.roles !== '[]'">
<a style="cursor:pointer"
ng-click="setInterpreterRunningOption(setting.id, 'shared', 'scoped')">
Per User
</a>
</li>
</ul>
</span>
in
<span class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
data-toggle="dropdown"
ng-disabled="!valueform.$visible
|| getInterpreterRunningOption(setting.id) === 'Globally'">
<span ng-if="getInterpreterRunningOption(setting.id) !== 'Per User'">
{{getPerNoteOption(setting.id)}}
</span>
<span ng-if="getInterpreterRunningOption(setting.id) === 'Per User'">
{{getPerUserOption(setting.id)}}
</span>
<span class="caret"></span>
</button>
<ul class="dropdown-menu" role="menu">
<li
ng-if="getInterpreterRunningOption(setting.id) === 'Globally'">
<a style="cursor:pointer"
tooltip="Single interpreter instance are shared across notes"
ng-click="setPerNoteOption(setting.id, 'shared')">
shared per note
</a>
</li>
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per Note'"
tooltip="Separate Interpreter instance for each note"
ng-click="setPerNoteOption(setting.id, 'scoped')">
scoped per note
</a>
</li>
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per User'"
tooltip="Separate Interpreter instance for each note"
ng-click="setPerUserOption(setting.id, 'scoped')">
scoped per user
</a>
</li>
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per Note'"
tooltip="Separate Interpreter process for each note"
ng-click="setPerNoteOption(setting.id, 'isolated')">
isolated per note
</a>
</li>
<li>
<a style="cursor:pointer"
ng-if="getInterpreterRunningOption(setting.id) === 'Per User'"
tooltip="Separate Interpreter process for each note"
ng-click="setPerUserOption(setting.id, 'isolated')">
isolated per user
</a>
</li>
</ul>
</span>
process.
<span ng-if="getInterpreterRunningOption(setting.id) === 'Per User' && ticket.ticket !== 'anonymous' && ticket.roles !== '[]'">
<span ng-if="getPerNoteOption(setting.id) === 'shared'">
<button type="button" class="btn btn-default btn-xs"
ng-click="setPerNoteOption(setting.id, 'scoped')"
ng-disabled="!valueform.$visible"
data-toggle="dropdown">
<i class="fa fa-plus"></i>
</button>
</span>
</span>
</div>
<div class="col-md-6">
&nbsp;
</div>
</div>
<div class="row interpreter" style="margin-top: 6px;">
<div class="col-md-6">
<span ng-if="getInterpreterRunningOption(setting.id) === 'Per User' && ticket.ticket !== 'anonymous' && ticket.roles !== '[]'">
<span ng-if="getPerNoteOption(setting.id) !== 'shared'">
<span class="hidden-xs" style="padding-left: 190px;">And </span>
<span class="visible-xs" style="padding-left: 0px;">And </span>
<span class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
data-toggle="dropdown"
ng-disabled="true">
<span>
Per Note
</span>
<span class="caret"></span>
</button>
</span>
in
<span class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle"
data-toggle="dropdown"
ng-disabled="!valueform.$visible">
<span>
{{getPerNoteOption(setting.id)}}
</span>
<span class="caret"></span>
</button>
<ul class="dropdown-menu" role="menu">
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter instance for each note"
ng-click="setPerNoteOption(setting.id, 'scoped')">
scoped per note
</a>
</li>
<li>
<a style="cursor:pointer"
tooltip="Separate Interpreter process for each note"
ng-click="setPerNoteOption(setting.id, 'isolated')">
isolated per note
</a>
</li>
</ul>
</span>
process.
<button type="button" class="btn btn-default btn-xs"
ng-disabled="!valueform.$visible"
ng-click="setPerNoteOption(setting.id, 'shared')"
data-toggle="dropdown">
<i class="fa fa-minus"></i>
</button>
</span>
</span>
</div>
<div class="col-md-6">
&nbsp;
</div>
</div>
</div>
</div>
<div class="row interpreter" style="margin-top: 5px;">

99
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -647,6 +647,42 @@
$scope.permissions.writers = angular.element('#selectWriters').val();
}
$scope.restartInterpreter = function(interpeter) {
var thisConfirm = BootstrapDialog.confirm({
closable: false,
closeByBackdrop: false,
closeByKeyboard: false,
title: '',
message: 'Do you want to restart ' + interpeter.name + ' interpreter?',
callback: function(result) {
if (result) {
var payload = {
'noteId': $scope.note.id
};
thisConfirm.$modalFooter.find('button').addClass('disabled');
thisConfirm.$modalFooter.find('button:contains("OK")')
.html('<i class="fa fa-circle-o-notch fa-spin"></i> Saving Setting');
$http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/restart/' + interpeter.id, payload)
.success(function(data, status, headers, config) {
var index = _.findIndex($scope.interpreterSettings, {'id': interpeter.id});
$scope.interpreterSettings[index] = data.body;
thisConfirm.close();
}).error(function(data, status, headers, config) {
thisConfirm.close();
console.log('Error %o %o', status, data.message);
BootstrapDialog.show({
title: 'Error restart interpreter.',
message: data.message
});
});
return false;
}
}
});
};
$scope.savePermissions = function() {
convertPermissionsToArray();
$http.put(baseUrlSrv.getRestApiBase() + '/notebook/' + $scope.note.id + '/permissions',
@ -660,69 +696,6 @@
message: 'Owners : ' + $scope.permissions.owners + '\n\n' + 'Readers : ' +
$scope.permissions.readers + '\n\n' + 'Writers : ' + $scope.permissions.writers
});
<<<<<<< HEAD
}
}
});
} else {
$scope.showPermissions = false;
}
};
function convertPermissionsToArray() {
$scope.permissions.owners = angular.element('#selectOwners').val();
$scope.permissions.readers = angular.element('#selectReaders').val();
$scope.permissions.writers = angular.element('#selectWriters').val();
}
$scope.restartInterpreter = function(interpeter) {
var thisConfirm = BootstrapDialog.confirm({
closable: false,
closeByBackdrop: false,
closeByKeyboard: false,
title: '',
message: 'Do you want to restart ' + interpeter.name + ' interpreter?',
callback: function(result) {
if (result) {
var payload = {
'noteId': $scope.note.id
};
thisConfirm.$modalFooter.find('button').addClass('disabled');
thisConfirm.$modalFooter.find('button:contains("OK")')
.html('<i class="fa fa-circle-o-notch fa-spin"></i> Saving Setting');
$http.put(baseUrlSrv.getRestApiBase() + '/interpreter/setting/restart/' + interpeter.id, payload)
.success(function(data, status, headers, config) {
var index = _.findIndex($scope.interpreterSettings, {'id': interpeter.id});
$scope.interpreterSettings[index] = data.body;
thisConfirm.close();
}).error(function(data, status, headers, config) {
thisConfirm.close();
console.log('Error %o %o', status, data.message);
BootstrapDialog.show({
title: 'Error restart interpreter.',
message: data.message
});
});
return false;
}
}
});
};
$scope.savePermissions = function() {
convertPermissionsToArray();
$http.put(baseUrlSrv.getRestApiBase() + '/notebook/' + $scope.note.id + '/permissions',
$scope.permissions, {withCredentials: true}).
success(function(data, status, headers, config) {
getPermissions(function() {
console.log('Note permissions %o saved', $scope.permissions);
BootstrapDialog.alert({
closable: true,
title: 'Permissions Saved Successfully!!!',
message: 'Owners : ' + $scope.permissions.owners + '\n\n' + 'Readers : ' +
$scope.permissions.readers + '\n\n' + 'Writers : ' + $scope.permissions.writers
$scope.showPermissions = false;
});
}).

14
zeppelin-web/src/components/navbar/navbar.controller.js
View file

@ -57,6 +57,7 @@
}
function initController() {
$scope.isDrawNavbarNotebookList = false;
angular.element('#notebook-list').perfectScrollbar({suppressScrollX: true});
angular.element(document).click(function() {
@ -122,6 +123,19 @@
$scope.$on('loginSuccess', function(event, param) {
loadNotes();
});
/*
** Performance optimization for Browser Render.
*/
angular.element(document).ready(function() {
angular.element('.notebook-list-dropdown').on('show.bs.dropdown', function() {
$scope.isDrawNavbarNotebookList = true;
});
angular.element('.notebook-list-dropdown').on('hide.bs.dropdown', function() {
$scope.isDrawNavbarNotebookList = false;
});
});
}
})();

6
zeppelin-web/src/components/navbar/navbar.html
View file

@ -25,12 +25,12 @@ limitations under the License.
<div class="collapse navbar-collapse" ng-controller="NavCtrl as navbar">
<ul class="nav navbar-nav" ng-if="ticket">
<li class="dropdown" dropdown>
<a href="#" class="dropdown-toggle" dropdown-toggle>Notebook <span class="caret"></span></a>
<li class="dropdown notebook-list-dropdown" dropdown>
<a href="#" class="dropdown-toggle" data-toggle="dropdown" dropdown-toggle>Notebook <span class="caret"></span></a>
<ul class="dropdown-menu navbar-dropdown-maxHeight" role="menu">
<li><a href="" data-toggle="modal" data-target="#noteNameModal"><i class="fa fa-plus"></i> Create new note</a></li>
<li class="divider"></li>
<div id="notebook-list" class="scrollbar-container">
<div id="notebook-list" class="scrollbar-container" ng-if="isDrawNavbarNotebookList">
<li class="filter-names" ng-include="'components/filterNoteNames/filter-note-names.html'"></li>
<li ng-repeat="note in navbar.notes.root.children | filter:query.q | orderBy:navbar.arrayOrderingSrv.notebookListOrdering track by $index"
ng-class="{'active' : navbar.isActive(note.id)}" ng-include="'components/navbar/navbar-notebookList-elem.html'">

137
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
View file

@ -121,6 +121,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
private DependencyResolver depResolver;
private boolean shiroEnabled;
private Map<String, String> env = new HashMap<>();
private Interpreter devInterpreter;
@ -128,18 +130,18 @@ public class InterpreterFactory implements InterpreterGroupFactory {
public InterpreterFactory(ZeppelinConfiguration conf,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appEventListener, DependencyResolver depResolver)
throws InterpreterException, IOException, RepositoryException {
ApplicationEventListener appEventListener, DependencyResolver depResolver,
boolean shiroEnabled) throws InterpreterException, IOException, RepositoryException {
this(conf, new InterpreterOption(true), angularObjectRegistryListener,
remoteInterpreterProcessListener, appEventListener, depResolver);
remoteInterpreterProcessListener, appEventListener, depResolver, shiroEnabled);
}
public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appEventListener, DependencyResolver depResolver)
throws InterpreterException, IOException, RepositoryException {
ApplicationEventListener appEventListener, DependencyResolver depResolver,
boolean shiroEnabled) throws InterpreterException, IOException, RepositoryException {
this.conf = conf;
this.defaultOption = defaultOption;
this.angularObjectRegistryListener = angularObjectRegistryListener;
@ -147,6 +149,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
this.interpreterRepositories = depResolver.getRepos();
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.appEventListener = appEventListener;
this.shiroEnabled = shiroEnabled;
String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS);
interpreterClassList = replsConf.split(",");
String groupOrder = conf.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER);
@ -157,6 +160,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
gson = builder.create();
init();
logger.info("shiroEnabled: {}", shiroEnabled);
}
private void init() throws InterpreterException, IOException, RepositoryException {
@ -698,16 +703,18 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return interpreterGroup;
}
public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String noteId) {
if (interpreterSetting.getOption().isPerNoteProcess()) {
public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user,
String noteId) {
InterpreterOption option = interpreterSetting.getOption();
if (option.isProcess()) {
interpreterSetting.closeAndRemoveInterpreterGroup(noteId);
} else if (interpreterSetting.getOption().isPerNoteSession()) {
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(noteId);
interpreterGroup.close(noteId);
interpreterGroup.destroy(noteId);
} else if (option.isSession()) {
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
String key = getInterpreterInstanceKey(user, noteId, interpreterSetting);
interpreterGroup.close(key);
interpreterGroup.destroy(key);
synchronized (interpreterGroup) {
interpreterGroup.remove(noteId);
interpreterGroup.remove(key);
interpreterGroup.notifyAll(); // notify createInterpreterForNote()
}
logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(),
@ -715,9 +722,9 @@ public class InterpreterFactory implements InterpreterGroupFactory {
}
}
public void createInterpretersForNote(InterpreterSetting interpreterSetting, String noteId,
String key) {
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(noteId);
public void createInterpretersForNote(InterpreterSetting interpreterSetting, String user,
String noteId, String key) {
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
InterpreterOption option = interpreterSetting.getOption();
Properties properties = interpreterSetting.getProperties();
if (option.isExistingProcess) {
@ -861,8 +868,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
}
}
private void putNoteInterpreterSettingBinding(String noteId, List<String> settingList)
throws IOException {
private void putNoteInterpreterSettingBinding(String user, String noteId,
List<String> settingList) throws IOException {
List<String> unBindedSettings = new LinkedList<>();
synchronized (interpreterSettings) {
@ -879,18 +886,18 @@ public class InterpreterFactory implements InterpreterGroupFactory {
for (String settingId : unBindedSettings) {
InterpreterSetting setting = get(settingId);
removeInterpretersForNote(setting, noteId);
removeInterpretersForNote(setting, user, noteId);
}
}
}
public void removeNoteInterpreterSettingBinding(String noteId) {
public void removeNoteInterpreterSettingBinding(String user, String noteId) {
synchronized (interpreterSettings) {
List<String> settingIds = (interpreterBindings.containsKey(noteId) ?
interpreterBindings.remove(noteId) :
Collections.<String>emptyList());
for (String settingId : settingIds) {
this.removeInterpretersForNote(get(settingId), noteId);
this.removeInterpretersForNote(get(settingId), user, noteId);
}
}
}
@ -944,16 +951,12 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return noteId == null ? false : true;
}
private boolean isNotSharedInterpreter(InterpreterOption intpOption) {
return intpOption.isPerNoteSession() || intpOption.isExistingProcess();
}
public void restart(String settingId, String noteId) {
InterpreterSetting setting = interpreterSettings.get(settingId);
if (noteIdIsExist(noteId) && isNotSharedInterpreter(setting.getOption())) {
removeInterpretersForNote(setting, noteId);
InterpreterSetting intpsetting = interpreterSettings.get(settingId);
if (intpsetting != null &&
noteIdIsExist(noteId) &&
intpsetting.getOption().isProcess()) {
intpsetting.closeAndRemoveInterpreterGroup(noteId);
return;
}
restart(settingId);
@ -964,8 +967,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
InterpreterSetting intpsetting = interpreterSettings.get(id);
// Check if dependency in specified path is changed
// If it did, overwrite old dependency jar with new one
copyDependenciesFromLocalPath(intpsetting);
if (intpsetting != null) {
copyDependenciesFromLocalPath(intpsetting);
stopJobAllInterpreter(intpsetting);
@ -1112,8 +1115,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
* @param ids InterpreterSetting id list
* @throws IOException
*/
public void setInterpreters(String noteId, List<String> ids) throws IOException {
putNoteInterpreterSettingBinding(noteId, ids);
public void setInterpreters(String user, String noteId, List<String> ids) throws IOException {
putNoteInterpreterSettingBinding(user, noteId, ids);
}
public List<String> getInterpreters(String noteId) {
@ -1138,7 +1141,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return settings;
}
public void closeNote(String noteId) {
public void closeNote(String user, String noteId) {
// close interpreters in this note session
List<InterpreterSetting> settings = getInterpreterSettings(noteId);
if (settings == null || settings.size() == 0) {
@ -1147,28 +1150,37 @@ public class InterpreterFactory implements InterpreterGroupFactory {
logger.info("closeNote: {}", noteId);
for (InterpreterSetting setting : settings) {
removeInterpretersForNote(setting, noteId);
removeInterpretersForNote(setting, user, noteId);
}
}
private String getInterpreterInstanceKey(String noteId, InterpreterSetting setting) {
if (setting.getOption().isExistingProcess()) {
return Constants.EXISTING_PROCESS;
} else if (setting.getOption().isPerNoteSession() || setting.getOption().isPerNoteProcess()) {
return noteId;
} else {
return SHARED_SESSION;
}
}
private List<Interpreter> createOrGetInterpreterList(String noteId, InterpreterSetting setting) {
InterpreterGroup interpreterGroup = setting.getInterpreterGroup(noteId);
synchronized (interpreterGroup) {
String key = getInterpreterInstanceKey(noteId, setting);
if (!interpreterGroup.containsKey(key)) {
createInterpretersForNote(setting, noteId, key);
private String getInterpreterInstanceKey(String user, String noteId, InterpreterSetting setting) {
InterpreterOption option = setting.getOption();
String key;
if (option.isExistingProcess()) {
key = Constants.EXISTING_PROCESS;
} else if (!option.perNoteShared()) {
key = noteId;
if (shiroEnabled && !option.perUserShared()) {
key = user + ":" + key;
}
return interpreterGroup.get(getInterpreterInstanceKey(noteId, setting));
} else {
key = SHARED_SESSION;
}
logger.debug("Interpreter instance key: {}", key);
return key;
}
private List<Interpreter> createOrGetInterpreterList(String user, String noteId,
InterpreterSetting setting) {
InterpreterGroup interpreterGroup = setting.getInterpreterGroup(user, noteId);
synchronized (interpreterGroup) {
String key = getInterpreterInstanceKey(user, noteId, setting);
if (!interpreterGroup.containsKey(key)) {
createInterpretersForNote(setting, user, noteId, key);
}
return interpreterGroup.get(getInterpreterInstanceKey(user, noteId, setting));
}
}
@ -1209,14 +1221,15 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return null;
}
private Interpreter getInterpreter(String noteId, InterpreterSetting setting, String name) {
private Interpreter getInterpreter(String user, String noteId, InterpreterSetting setting,
String name) {
Preconditions.checkNotNull(noteId, "noteId should be not null");
Preconditions.checkNotNull(setting, "setting should be not null");
Preconditions.checkNotNull(name, "name should be not null");
String className;
if (null != (className = getInterpreterClassFromInterpreterSetting(setting, name))) {
List<Interpreter> interpreterGroup = createOrGetInterpreterList(noteId, setting);
List<Interpreter> interpreterGroup = createOrGetInterpreterList(user, noteId, setting);
for (Interpreter interpreter : interpreterGroup) {
if (className.equals(interpreter.getClassName())) {
return interpreter;
@ -1226,7 +1239,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return null;
}
public Interpreter getInterpreter(String noteId, String replName) {
public Interpreter getInterpreter(String user, String noteId, String replName) {
List<InterpreterSetting> settings = getInterpreterSettings(noteId);
InterpreterSetting setting;
Interpreter interpreter;
@ -1239,7 +1252,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
// get default settings (first available)
// TODO(jl): Fix it in case of returning null
InterpreterSetting defaultSettings = getDefaultInterpreterSetting(settings);
return createOrGetInterpreterList(noteId, defaultSettings).get(0);
return createOrGetInterpreterList(user, noteId, defaultSettings).get(0);
}
String[] replNameSplit = replName.split("\\.");
@ -1252,7 +1265,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
setting = getInterpreterSettingByGroup(settings, group);
if (null != setting) {
interpreter = getInterpreter(noteId, setting, name);
interpreter = getInterpreter(user, noteId, setting, name);
if (null != interpreter) {
return interpreter;
@ -1267,7 +1280,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
// TODO(jl): Handle with noteId to support defaultInterpreter per note.
setting = getDefaultInterpreterSetting(settings);
interpreter = getInterpreter(noteId, setting, replName);
interpreter = getInterpreter(user, noteId, setting, replName);
if (null != interpreter) {
return interpreter;
@ -1278,7 +1291,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
setting = getInterpreterSettingByGroup(settings, replName);
if (null != setting) {
List<Interpreter> interpreters = createOrGetInterpreterList(noteId, setting);
List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, setting);
if (null != interpreters) {
return interpreters.get(0);
}
@ -1287,7 +1300,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
// Support the legacy way to use it
for (InterpreterSetting s : settings) {
if (s.getGroup().equals(replName)) {
List<Interpreter> interpreters = createOrGetInterpreterList(noteId, s);
List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, s);
if (null != interpreters) {
return interpreters.get(0);
}
@ -1349,8 +1362,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
this.env = env;
}
public Map<String, Object> getEditorSetting(String noteId, String replName) {
Interpreter intp = getInterpreter(noteId, replName);
public Map<String, Object> getEditorSetting(String user, String noteId, String replName) {
Interpreter intp = getInterpreter(user, noteId, replName);
Map<String, Object> editor = Maps.newHashMap(
ImmutableMap.<String, Object>builder()
.put("language", "text").build());

74
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java
View file

@ -17,18 +17,25 @@
package org.apache.zeppelin.interpreter;
import com.google.common.base.Preconditions;
import java.util.List;
/**
*
*/
public class InterpreterOption {
public static final transient String SHARED = "shared";
public static final transient String SCOPED = "scoped";
public static final transient String ISOLATED = "isolated";
boolean remote;
String host = null;
int port = -1;
boolean perNoteSession;
boolean perNoteProcess;
String perNote;
String perUser;
boolean isExistingProcess;
boolean setPermission;
List<String> users;
@ -62,11 +69,21 @@ public class InterpreterOption {
}
public InterpreterOption() {
remote = false;
this(false);
}
public InterpreterOption(boolean remote) {
this(remote, SHARED, SHARED);
}
public InterpreterOption(boolean remote, String perUser, String perNote) {
Preconditions.checkNotNull(remote);
Preconditions.checkNotNull(perUser);
Preconditions.checkNotNull(perNote);
this.remote = remote;
this.perUser = perUser;
this.perNote = perNote;
}
public boolean isRemote() {
@ -77,14 +94,6 @@ public class InterpreterOption {
this.remote = remote;
}
public boolean isPerNoteSession() {
return perNoteSession;
}
public void setPerNoteSession(boolean perNoteSession) {
this.perNoteSession = perNoteSession;
}
public String getHost() {
return host;
}
@ -93,11 +102,44 @@ public class InterpreterOption {
return port;
}
public boolean isPerNoteProcess() {
return perNoteProcess;
public boolean perUserShared() {
return SHARED.equals(perUser);
}
public void setPerNoteProcess(boolean perNoteProcess) {
this.perNoteProcess = perNoteProcess;
public boolean perUserScoped() {
return SCOPED.equals(perUser);
}
public boolean perUserIsolated() {
return ISOLATED.equals(perUser);
}
public boolean perNoteShared() {
return SHARED.equals(perNote);
}
public boolean perNoteScoped() {
return SCOPED.equals(perNote);
}
public boolean perNoteIsolated() {
return ISOLATED.equals(perNote);
}
public boolean isProcess() {
return perUserIsolated() || perNoteIsolated();
}
public boolean isSession() {
return perUserScoped() || perNoteScoped();
}
public void setPerNote(String perNote) {
this.perNote = perNote;
}
public void setPerUser(String perUser) {
this.perUser = perUser;
}
}

77
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
View file

@ -24,8 +24,11 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.gson.annotations.SerializedName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.zeppelin.dep.Dependency;
@ -35,6 +38,7 @@ import static org.apache.zeppelin.notebook.utility.IdHashes.generateId;
* Interpreter settings
*/
public class InterpreterSetting {
private static final Logger logger = LoggerFactory.getLogger(InterpreterSetting.class);
private static final String SHARED_PROCESS = "shared_process";
private String id;
private String name;
@ -51,13 +55,19 @@ public class InterpreterSetting {
@Deprecated private transient InterpreterGroupFactory interpreterGroupFactory;
public InterpreterSetting() {
private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock;
private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock;
public InterpreterSetting() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
interpreterGroupReadLock = lock.readLock();
interpreterGroupWriteLock = lock.writeLock();
}
public InterpreterSetting(String id, String name, String group,
List<InterpreterInfo> interpreterInfos, Properties properties, List<Dependency> dependencies,
InterpreterOption option, String path) {
this();
this.id = id;
this.name = name;
this.group = group;
@ -96,40 +106,59 @@ public class InterpreterSetting {
return group;
}
private String getInterpreterProcessKey(String noteId) {
private String getInterpreterProcessKey(String user, String noteId) {
InterpreterOption option = getOption();
String key;
if (getOption().isExistingProcess) {
return Constants.EXISTING_PROCESS;
} else if (getOption().isPerNoteProcess()) {
return noteId;
key = Constants.EXISTING_PROCESS;
} else if (getOption().isProcess()) {
key = (option.perUserIsolated() ? user : "") + ":" + (option.perNoteIsolated() ? noteId : "");
} else {
return SHARED_PROCESS;
key = SHARED_PROCESS;
}
logger.debug("getInterpreterProcessKey: {}", key);
return key;
}
public InterpreterGroup getInterpreterGroup(String noteId) {
String key = getInterpreterProcessKey(noteId);
synchronized (interpreterGroupRef) {
if (!interpreterGroupRef.containsKey(key)) {
String interpreterGroupId = getId() + ":" + key;
InterpreterGroup intpGroup =
interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption());
interpreterGroupRef.put(key, intpGroup);
}
public InterpreterGroup getInterpreterGroup(String user, String noteId) {
String key = getInterpreterProcessKey(user, noteId);
if (!interpreterGroupRef.containsKey(key)) {
String interpreterGroupId = getId() + ":" + key;
InterpreterGroup intpGroup =
interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption());
interpreterGroupWriteLock.lock();
interpreterGroupRef.put(key, intpGroup);
interpreterGroupWriteLock.unlock();
}
try {
interpreterGroupReadLock.lock();
return interpreterGroupRef.get(key);
} finally {
interpreterGroupReadLock.unlock();
}
}
public Collection<InterpreterGroup> getAllInterpreterGroups() {
synchronized (interpreterGroupRef) {
try {
interpreterGroupReadLock.lock();
return new LinkedList<>(interpreterGroupRef.values());
} finally {
interpreterGroupReadLock.unlock();
}
}
void closeAndRemoveInterpreterGroup(String noteId) {
String key = getInterpreterProcessKey(noteId);
InterpreterGroup groupToRemove;
synchronized (interpreterGroupRef) {
groupToRemove = interpreterGroupRef.remove(key);
String key = getInterpreterProcessKey("", noteId);
InterpreterGroup groupToRemove = null;
for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
if (intpKey.contains(key)) {
interpreterGroupWriteLock.lock();
groupToRemove = interpreterGroupRef.remove(intpKey);
interpreterGroupWriteLock.unlock();
}
}
if (groupToRemove != null) {
@ -139,11 +168,9 @@ public class InterpreterSetting {
}
void closeAndRmoveAllInterpreterGroups() {
synchronized (interpreterGroupRef) {
HashSet<String> groupsToRemove = new HashSet<>(interpreterGroupRef.keySet());
for (String key : groupsToRemove) {
closeAndRemoveInterpreterGroup(key);
}
HashSet<String> groupsToRemove = new HashSet<>(interpreterGroupRef.keySet());
for (String key : groupsToRemove) {
closeAndRemoveInterpreterGroup(key);
}
}

29
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -30,6 +30,7 @@ import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
import com.google.gson.Gson;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@ -286,8 +287,8 @@ public class Note implements Serializable, ParagraphJobListener {
* @param paragraphId ID of paragraph
* @return a paragraph that was deleted, or <code>null</code> otherwise
*/
public Paragraph removeParagraph(String paragraphId) {
removeAllAngularObjectInParagraph(paragraphId);
public Paragraph removeParagraph(String user, String paragraphId) {
removeAllAngularObjectInParagraph(user, paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
@ -350,8 +351,8 @@ public class Note implements Serializable, ParagraphJobListener {
if (index < 0 || index >= paragraphs.size()) {
if (throwWhenIndexIsOutOfBound) {
throw new IndexOutOfBoundsException(
"paragraph size is " + paragraphs.size() + " , index is " + index);
throw new IndexOutOfBoundsException("paragraph size is " + paragraphs.size() +
" , index is " + index);
} else {
return;
}
@ -437,6 +438,8 @@ public class Note implements Serializable, ParagraphJobListener {
}
if (p.getStatus().isRunning()) {
info.put("progress", String.valueOf(p.progress()));
} else {
info.put("progress", String.valueOf(100));
}
return info;
}
@ -460,6 +463,9 @@ public class Note implements Serializable, ParagraphJobListener {
*/
public void runAll() {
String cronExecutingUser = (String) getConfig().get("cronExecutingUser");
if (null == cronExecutingUser) {
cronExecutingUser = "anonymous";
}
synchronized (paragraphs) {
for (Paragraph p : paragraphs) {
if (!p.isEnabled()) {
@ -482,7 +488,8 @@ public class Note implements Serializable, ParagraphJobListener {
Paragraph p = getParagraph(paragraphId);
p.setListener(jobListenerFactory.getParagraphJobListener(this));
String requiredReplName = p.getRequiredReplName();
Interpreter intp = factory.getInterpreter(getId(), requiredReplName);
Interpreter intp = factory.getInterpreter(p.getUser(), getId(), requiredReplName);
if (intp == null) {
String intpExceptionMsg =
p.getJobName() + "'s Interpreter " + requiredReplName + " not found";
@ -494,6 +501,7 @@ public class Note implements Serializable, ParagraphJobListener {
throw intpException;
}
if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
p.setAuthenticationInfo(p.getAuthenticationInfo());
intp.getScheduler().submit(p);
}
}
@ -526,7 +534,7 @@ public class Note implements Serializable, ParagraphJobListener {
}
}
private void snapshotAngularObjectRegistry() {
private void snapshotAngularObjectRegistry(String user) {
angularObjects = new HashMap<>();
List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
@ -535,13 +543,13 @@ public class Note implements Serializable, ParagraphJobListener {
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(id);
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
}
}
private void removeAllAngularObjectInParagraph(String paragraphId) {
private void removeAllAngularObjectInParagraph(String user, String paragraphId) {
angularObjects = new HashMap<>();
List<InterpreterSetting> settings = factory.getInterpreterSettings(getId());
@ -550,7 +558,7 @@ public class Note implements Serializable, ParagraphJobListener {
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(id);
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
@ -580,8 +588,9 @@ public class Note implements Serializable, ParagraphJobListener {
}
public void persist(AuthenticationInfo subject) throws IOException {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
stopDelayedPersistTimer();
snapshotAngularObjectRegistry();
snapshotAngularObjectRegistry(subject.getUser());
index.updateIndexDoc(this);
repo.save(this, subject);
}

28
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -31,6 +31,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Sets;
@ -135,6 +137,7 @@ public class Notebook implements NoteEventListener {
* @throws IOException
*/
public Note createNote(AuthenticationInfo subject) throws IOException {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
note = createNote(replFactory.getDefaultInterpreterSettingList(), subject);
@ -158,7 +161,7 @@ public class Notebook implements NoteEventListener {
notes.put(note.getId(), note);
}
if (interpreterIds != null) {
bindInterpretersToNote(note.getId(), interpreterIds);
bindInterpretersToNote(subject.getUser(), note.getId(), interpreterIds);
}
if (subject != null && !"anonymous".equals(subject.getUser())) {
@ -252,7 +255,7 @@ public class Notebook implements NoteEventListener {
}
// Copy the interpreter bindings
List<String> boundInterpreterSettingsIds = getBindedInterpreterSettingsIds(sourceNote.getId());
bindInterpretersToNote(newNote.getId(), boundInterpreterSettingsIds);
bindInterpretersToNote(subject.getUser(), newNote.getId(), boundInterpreterSettingsIds);
List<Paragraph> paragraphs = sourceNote.getParagraphs();
for (Paragraph p : paragraphs) {
@ -264,7 +267,7 @@ public class Notebook implements NoteEventListener {
return newNote;
}
public void bindInterpretersToNote(String id, List<String> interpreterSettingIds)
public void bindInterpretersToNote(String user, String id, List<String> interpreterSettingIds)
throws IOException {
Note note = getNote(id);
if (note != null) {
@ -275,7 +278,7 @@ public class Notebook implements NoteEventListener {
}
}
replFactory.setInterpreters(note.getId(), interpreterSettingIds);
replFactory.setInterpreters(user, note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreters(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
@ -306,18 +309,21 @@ public class Notebook implements NoteEventListener {
}
public void removeNote(String id, AuthenticationInfo subject) {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
Note note;
synchronized (notes) {
note = notes.remove(id);
}
replFactory.removeNoteInterpreterSettingBinding(id);
replFactory.removeNoteInterpreterSettingBinding(subject.getUser(), id);
notebookIndex.deleteIndexDocs(note);
notebookAuthorization.removeNote(id);
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : replFactory.get()) {
AngularObjectRegistry registry = settings.getInterpreterGroup(id).getAngularObjectRegistry();
AngularObjectRegistry registry =
settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
@ -437,7 +443,7 @@ public class Notebook implements NoteEventListener {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = replFactory.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(note.getId());
InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();
@ -533,11 +539,11 @@ public class Notebook implements NoteEventListener {
return noteList;
}
}
public List<Note> getAllNotes(AuthenticationInfo subject) {
public List<Note> getAllNotes(HashSet<String> userAndRoles) {
final Set<String> entities = Sets.newHashSet();
if (subject != null) {
entities.add(subject.getUser());
if (userAndRoles != null) {
entities.addAll(userAndRoles);
}
synchronized (notes) {

13
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -105,6 +105,10 @@ public class Paragraph extends Job implements Serializable, Cloneable {
+ new Random(System.currentTimeMillis()).nextInt();
}
public String getUser() {
return user;
}
public String getText() {
return text;
}
@ -193,7 +197,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
public Interpreter getRepl(String name) {
return factory.getInterpreter(note.getId(), name);
return factory.getInterpreter(user, note.getId(), name);
}
public Interpreter getCurrentRepl() {
@ -442,8 +446,8 @@ public class Paragraph extends Job implements Serializable, Cloneable {
if (!factory.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(note.getId()).getResourcePool();
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<InterpreterContextRunner>();
@ -582,6 +586,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
}
private boolean isValidInterpreter(String replName) {
return factory.getInterpreter(note.getId(), replName) != null;
return factory.getInterpreter("",
note.getId(), replName) != null;
}
}

47
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
View file

@ -27,6 +27,8 @@ import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -50,6 +52,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
private VFSNotebookRepo notebookRepo;
private Notebook notebook;
private HeliumApplicationFactory heliumAppFactory;
private AuthenticationInfo anonymous;
@Before
public void setUp() throws Exception {
@ -82,7 +85,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
heliumAppFactory = new HeliumApplicationFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
factory = new InterpreterFactory(conf,
new InterpreterOption(true), null, null, heliumAppFactory, depResolver);
new InterpreterOption(true), null, null, heliumAppFactory, depResolver, false);
HashMap<String, String> env = new HashMap<String, String>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
factory.setEnv(env);
@ -98,11 +101,13 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
this,
search,
notebookAuthorization,
null);
new Credentials(false, null));
heliumAppFactory.setNotebook(notebook);
notebook.addNotebookEventListener(heliumAppFactory);
anonymous = new AuthenticationInfo("anonymous");
}
@After
@ -131,13 +136,14 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
HeliumTestApplication.class.getName(),
new String[][]{});
Note note1 = notebook.createNote(null);
factory.setInterpreters(note1.getId(),factory.getDefaultInterpreterSettingList());
Note note1 = notebook.createNote(anonymous);
factory.setInterpreters("user", note1.getId(),factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
// make sure interpreter process running
p1.setText("%mock1 job");
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield();
@ -162,7 +168,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
// clean
heliumAppFactory.unload(p1, appId);
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@Test
@ -175,13 +181,14 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
HeliumTestApplication.class.getName(),
new String[][]{});
Note note1 = notebook.createNote(null);
factory.setInterpreters(note1.getId(), factory.getDefaultInterpreterSettingList());
Note note1 = notebook.createNote(anonymous);
factory.setInterpreters("user", note1.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
// make sure interpreter process running
p1.setText("%mock1 job");
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield();
@ -193,13 +200,13 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
}
// when remove paragraph
note1.removeParagraph(p1.getId());
note1.removeParagraph("user", p1.getId());
// then
assertEquals(ApplicationState.Status.UNLOADED, app.getStatus());
// clean
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@ -213,13 +220,14 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
HeliumTestApplication.class.getName(),
new String[][]{});
Note note1 = notebook.createNote(null);
notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList());
Note note1 = notebook.createNote(anonymous);
notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note1.addParagraph();
// make sure interpreter process running
p1.setText("%mock1 job");
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield();
@ -231,19 +239,19 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
}
// when unbind interpreter
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
notebook.bindInterpretersToNote("user", note1.getId(), new LinkedList<String>());
// then
assertEquals(ApplicationState.Status.UNLOADED, app.getStatus());
// clean
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@Test
public void testInterpreterUnbindOfNullReplParagraph() throws IOException {
// create note
Note note1 = notebook.createNote(null);
Note note1 = notebook.createNote(anonymous);
// add paragraph with invalid magic
Paragraph p1 = note1.addParagraph();
@ -255,10 +263,10 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
// Unbind all interpreter from note
// NullPointerException shouldn't occur here
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
notebook.bindInterpretersToNote("user", note1.getId(), new LinkedList<String>());
// remove note
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@ -272,8 +280,8 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
HeliumTestApplication.class.getName(),
new String[][]{});
Note note1 = notebook.createNote(null);
notebook.bindInterpretersToNote(note1.getId(), factory.getDefaultInterpreterSettingList());
Note note1 = notebook.createNote(anonymous);
notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {
@ -286,6 +294,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
// make sure interpreter process running
p1.setText("%mock1 job");
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield();
assertEquals(0, p1.getAllApplicationStates().size());
@ -307,7 +316,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
assertEquals(ApplicationState.Status.UNLOADED, app.getStatus());
// clean
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@Override

57
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
View file

@ -33,6 +33,8 @@ import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.notebook.repo.zeppelinhub.security.Authentication;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
@ -87,7 +89,7 @@ public class InterpreterFactoryTest {
conf = new ZeppelinConfiguration();
schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
context = new InterpreterContext("note", "id", "title", "text", null, null, null, null, null, null, null);
SearchService search = mock(SearchService.class);
@ -114,8 +116,8 @@ public class InterpreterFactoryTest {
// mock1Setting = factory.createNewSetting("mock11", "mock1", new ArrayList<Dependency>(), new InterpreterOption(false), new Properties());
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("sharedProcess");
factory.createInterpretersForNote(mock1Setting, "sharedProcess", "session");
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
// get interpreter
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
@ -125,12 +127,12 @@ public class InterpreterFactoryTest {
// restart interpreter
factory.restart(mock1Setting.getId());
assertNull(mock1Setting.getInterpreterGroup("sharedProcess").get("session"));
assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session"));
}
@Test
public void testRemoteRepl() throws Exception {
factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver);
factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false);
List<InterpreterSetting> all = factory.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
@ -139,8 +141,8 @@ public class InterpreterFactoryTest {
break;
}
}
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("sharedProcess");
factory.createInterpretersForNote(mock1Setting, "sharedProcess", "session");
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
// get interpreter
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
assertTrue(interpreterGroup.get("session").get(0) instanceof LazyOpenInterpreter);
@ -186,13 +188,13 @@ public class InterpreterFactoryTest {
factory.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties());
assertEquals(numInterpreters + 1, factory.get().size());
InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver);
InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver, false);
assertEquals(numInterpreters + 1, factory2.get().size());
}
@Test
public void testInterpreterAliases() throws IOException, RepositoryException {
factory = new InterpreterFactory(conf, null, null, null, depResolver);
factory = new InterpreterFactory(conf, null, null, null, depResolver, false);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null);
factory.add("group1", new ArrayList<InterpreterInfo>(){{
@ -205,15 +207,38 @@ public class InterpreterFactoryTest {
final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
final InterpreterSetting setting2 = factory.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
factory.setInterpreters("note", new ArrayList<String>() {{
factory.setInterpreters("user", "note", new ArrayList<String>() {{
add(setting1.getId());
add(setting2.getId());
}});
assertEquals("className1", factory.getInterpreter("note", "test-group1").getClassName());
assertEquals("className1", factory.getInterpreter("note", "group1").getClassName());
assertEquals("className1", factory.getInterpreter("user1", "note", "test-group1").getClassName());
assertEquals("className1", factory.getInterpreter("user1", "note", "group1").getClassName());
}
@Test
public void testMultiUser() throws IOException, RepositoryException {
factory = new InterpreterFactory(conf, null, null, null, depResolver, true);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
factory.add("group1", new ArrayList<InterpreterInfo>(){{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), new Properties(), "/path1");
InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED);
final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new Properties());
factory.setInterpreters("user1", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
factory.setInterpreters("user2", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
assertNotEquals(factory.getInterpreter("user1", "note", "test-group1"), factory.getInterpreter("user2", "note", "test-group1"));
}
@Test
public void testInvalidInterpreterSettingName() {
try {
@ -233,19 +258,19 @@ public class InterpreterFactoryTest {
intpIds.add(intpSetting.getId());
}
}
Note note = notebook.createNote(intpIds, null);
Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous"));
// get editor setting from interpreter-setting.json
Map<String, Object> editor = factory.getEditorSetting(note.getId(), "mock11");
Map<String, Object> editor = factory.getEditorSetting("user1", note.getId(), "mock11");
assertEquals("java", editor.get("language"));
// when interpreter is not loaded via interpreter-setting.json
// or editor setting doesn't exit
editor = factory.getEditorSetting(note.getId(), "mock1");
editor = factory.getEditorSetting("user1", note.getId(), "mock1");
assertEquals(null, editor.get("language"));
// when interpreter is not bound to note
editor = factory.getEditorSetting(note.getId(), "mock2");
editor = factory.getEditorSetting("user1", note.getId(), "mock2");
assertEquals("text", editor.get("language"));
}
}

88
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
View file

@ -60,7 +60,7 @@ public class NoteInterpreterLoaderTest {
MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
}
@After
@ -71,92 +71,92 @@ public class NoteInterpreterLoaderTest {
@Test
public void testGetInterpreter() throws IOException {
factory.setInterpreters("note", factory.getDefaultInterpreterSettingList());
factory.setInterpreters("user", "note", factory.getDefaultInterpreterSettingList());
// when there're no interpreter selection directive
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", null).getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", " ").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", " ").getClassName());
// when group name is omitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "mock11").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "mock11").getClassName());
// when 'name' is ommitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2").getClassName());
// when nothing is ommitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1.mock1").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "group1.mock11").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2.mock2").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1.mock1").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName());
factory.closeNote("note");
factory.closeNote("user", "note");
}
@Test
public void testNoteSession() throws IOException {
factory.setInterpreters("noteA", factory.getDefaultInterpreterSettingList());
factory.getInterpreterSettings("noteA").get(0).getOption().setPerNoteSession(true);
factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList());
factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
factory.setInterpreters("noteB", factory.getDefaultInterpreterSettingList());
factory.getInterpreterSettings("noteB").get(0).getOption().setPerNoteSession(true);
factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList());
factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
// interpreters are not created before accessing it
assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA"));
assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB"));
assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
factory.getInterpreter("noteA", null).open();
factory.getInterpreter("noteB", null).open();
factory.getInterpreter("user", "noteA", null).open();
factory.getInterpreter("user", "noteB", null).open();
assertTrue(
factory.getInterpreter("noteA", null).getInterpreterGroup().getId().equals(
factory.getInterpreter("noteB", null).getInterpreterGroup().getId()));
factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
// interpreters are created after accessing it
assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA"));
assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB"));
assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
// when
factory.closeNote("noteA");
factory.closeNote("noteB");
factory.closeNote("user", "noteA");
factory.closeNote("user", "noteB");
// interpreters are destroyed after close
assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA"));
assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB"));
assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
}
@Test
public void testNotePerInterpreterProcess() throws IOException {
factory.setInterpreters("noteA", factory.getDefaultInterpreterSettingList());
factory.getInterpreterSettings("noteA").get(0).getOption().setPerNoteProcess(true);
factory.setInterpreters("user", "noteA", factory.getDefaultInterpreterSettingList());
factory.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
factory.setInterpreters("noteB", factory.getDefaultInterpreterSettingList());
factory.getInterpreterSettings("noteB").get(0).getOption().setPerNoteProcess(true);
factory.setInterpreters("user", "noteB", factory.getDefaultInterpreterSettingList());
factory.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
// interpreters are not created before accessing it
assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA"));
assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB"));
assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
factory.getInterpreter("noteA", null).open();
factory.getInterpreter("noteB", null).open();
factory.getInterpreter("user", "noteA", null).open();
factory.getInterpreter("user", "noteB", null).open();
// per note interpreter process
assertFalse(
factory.getInterpreter("noteA", null).getInterpreterGroup().getId().equals(
factory.getInterpreter("noteB", null).getInterpreterGroup().getId()));
factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
// interpreters are created after accessing it
assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA"));
assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB"));
assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
// when
factory.closeNote("noteA");
factory.closeNote("noteB");
factory.closeNote("user", "noteA");
factory.closeNote("user", "noteB");
// interpreters are destroyed after close
assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA"));
assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB"));
assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
}

14
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java
View file

@ -22,6 +22,7 @@ import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -58,9 +59,11 @@ public class NoteTest {
@Mock
InterpreterFactory interpreterFactory;
private AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
@Test
public void runNormalTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("spark"))).thenReturn(interpreter);
when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter);
when(interpreter.getScheduler()).thenReturn(scheduler);
String pText = "%spark sc.version";
@ -68,11 +71,12 @@ public class NoteTest {
Paragraph p = note.addParagraph();
p.setText(pText);
p.setAuthenticationInfo(anonymous);
note.run(p.getId());
ArgumentCaptor<Paragraph> pCaptor = ArgumentCaptor.forClass(Paragraph.class);
verify(scheduler, only()).submit(pCaptor.capture());
verify(interpreterFactory, only()).getInterpreter(anyString(), eq("spark"));
verify(interpreterFactory, only()).getInterpreter(anyString(), anyString(), eq("spark"));
assertEquals("Paragraph text", pText, pCaptor.getValue().getText());
}
@ -87,7 +91,7 @@ public class NoteTest {
@Test
public void addParagraphWithLastReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("spark"))).thenReturn(interpreter);
when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph();
@ -99,7 +103,7 @@ public class NoteTest {
@Test
public void insertParagraphWithLastReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("spark"))).thenReturn(interpreter);
when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("spark"))).thenReturn(interpreter);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph();
@ -111,7 +115,7 @@ public class NoteTest {
@Test
public void insertParagraphWithInvalidReplNameTest() {
when(interpreterFactory.getInterpreter(anyString(), eq("invalid"))).thenReturn(null);
when(interpreterFactory.getInterpreter(anyString(), anyString(), eq("invalid"))).thenReturn(null);
Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener);
Paragraph p1 = note.addParagraph();

281
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
View file

@ -65,6 +65,7 @@ public class NotebookTest implements JobListenerFactory{
private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
@Before
public void setUp() throws Exception {
@ -88,7 +89,7 @@ public class NotebookTest implements JobListenerFactory{
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
@ -106,8 +107,8 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testSelectingReplImplementation() throws IOException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
// run with default repl
Paragraph p1 = note.addParagraph();
@ -115,6 +116,7 @@ public class NotebookTest implements JobListenerFactory{
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
p1.setAuthenticationInfo(anonymous);
note.run(p1.getId());
while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield();
assertEquals("repl1: hello world", p1.getResult().message());
@ -123,10 +125,11 @@ public class NotebookTest implements JobListenerFactory{
Paragraph p2 = note.addParagraph();
p2.setConfig(config);
p2.setText("%mock2 hello world");
p2.setAuthenticationInfo(anonymous);
note.run(p2.getId());
while(p2.isTerminated()==false || p2.getResult()==null) Thread.yield();
assertEquals("repl2: hello world", p2.getResult().message());
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
@ -137,7 +140,7 @@ public class NotebookTest implements JobListenerFactory{
FileUtils.copyDirectory(srcDir, destDir);
// when load
notebook.reloadAllNotes(null);
notebook.reloadAllNotes(anonymous);
assertEquals(1, notebook.getAllNotes().size());
// then interpreter factory should be injected into all the paragraphs
@ -169,8 +172,8 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(notes.size(), 0);
// load copied notebook on memory when reloadAllNotes() is called
Note copiedNote = notebookRepo.get("2A94M5J1Z", null);
notebook.reloadAllNotes(null);
Note copiedNote = notebookRepo.get("2A94M5J1Z", anonymous);
notebook.reloadAllNotes(anonymous);
notes = notebook.getAllNotes();
assertEquals(notes.size(), 2);
assertEquals(notes.get(1).getId(), copiedNote.getId());
@ -188,14 +191,14 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(notes.size(), 2);
// delete notebook from notebook list when reloadAllNotes() is called
notebook.reloadAllNotes(null);
notebook.reloadAllNotes(anonymous);
notes = notebook.getAllNotes();
assertEquals(notes.size(), 0);
}
@Test
public void testPersist() throws IOException, SchedulerException, RepositoryException {
Note note = notebook.createNote(null);
Note note = notebook.createNote(anonymous);
// run with default repl
Paragraph p1 = note.addParagraph();
@ -203,14 +206,14 @@ public class NotebookTest implements JobListenerFactory{
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
note.persist(null);
note.persist(anonymous);
Notebook notebook2 = new Notebook(
conf, notebookRepo, schedulerFactory,
new InterpreterFactory(conf, null, null, null, depResolver), this, null, null, null);
new InterpreterFactory(conf, null, null, null, depResolver, false), this, null, null, null);
assertEquals(1, notebook2.getAllNotes().size());
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
@ -223,17 +226,18 @@ public class NotebookTest implements JobListenerFactory{
Set<String> owners = new HashSet<>();
owners.add("user1");
assertEquals(owners, notebook.getNotebookAuthorization().getOwners(note.getId()));
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testClearParagraphOutput() throws IOException, SchedulerException{
Note note = notebook.createNote(null);
Note note = notebook.createNote(anonymous);
Paragraph p1 = note.addParagraph();
Map config = p1.getConfig();
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
p1.setAuthenticationInfo(anonymous);
note.run(p1.getId());
while(p1.isTerminated() == false || p1.getResult() == null) Thread.yield();
@ -242,13 +246,13 @@ public class NotebookTest implements JobListenerFactory{
// clear paragraph output/result
note.clearParagraphOutput(p1.getId());
assertNull(p1.getResult());
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testRunAll() throws IOException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
// p1
Paragraph p1 = note.addParagraph();
@ -280,14 +284,14 @@ public class NotebookTest implements JobListenerFactory{
assertNull(p2.getResult());
assertEquals("repl1: p3", p3.getResult().message());
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testSchedule() throws InterruptedException, IOException {
// create a note and a paragraph
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p = note.addParagraph();
Map config = new HashMap<String, Object>();
@ -313,14 +317,14 @@ public class NotebookTest implements JobListenerFactory{
assertNotNull(dateFinished);
Thread.sleep(1 * 1000);
assertEquals(dateFinished, p.getDateFinished());
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p = note.addParagraph();
Map config = new HashMap<String, Object>();
@ -341,11 +345,11 @@ public class NotebookTest implements JobListenerFactory{
MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter)
((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock1")).getInnerInterpreter())
((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter())
.getInnerInterpreter()));
MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter)
((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock2")).getInnerInterpreter())
((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter())
.getInnerInterpreter()));
// wait until interpreters are started
@ -366,14 +370,14 @@ public class NotebookTest implements JobListenerFactory{
// make sure all paragraph has been executed
assertNotNull(p.getDateFinished());
assertNotNull(p2.getDateFinished());
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testExportAndImportNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
final Paragraph p = note.addParagraph();
String simpleText = "hello world";
@ -386,7 +390,7 @@ public class NotebookTest implements JobListenerFactory{
String exportedNoteJson = notebook.exportNote(note.getId());
Note importedNote = notebook.importNote(exportedNoteJson, "Title", null);
Note importedNote = notebook.importNote(exportedNoteJson, "Title", anonymous);
Paragraph p2 = importedNote.getParagraphs().get(0);
@ -403,16 +407,16 @@ public class NotebookTest implements JobListenerFactory{
Set<String> owners = new HashSet<>();
owners.add("user1");
assertEquals(owners, notebook.getNotebookAuthorization().getOwners(importedNote2.getId()));
notebook.removeNote(note.getId(), null);
notebook.removeNote(importedNote.getId(), null);
notebook.removeNote(importedNote2.getId(), null);
notebook.removeNote(note.getId(), anonymous);
notebook.removeNote(importedNote.getId(), anonymous);
notebook.removeNote(importedNote2.getId(), anonymous);
}
@Test
public void testCloneNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
final Paragraph p = note.addParagraph();
p.setText("hello world");
@ -420,7 +424,7 @@ public class NotebookTest implements JobListenerFactory{
while(p.isTerminated()==false || p.getResult()==null) Thread.yield();
p.setStatus(Status.RUNNING);
Note cloneNote = notebook.cloneNote(note.getId(), "clone note", null);
Note cloneNote = notebook.cloneNote(note.getId(), "clone note", anonymous);
Paragraph cp = cloneNote.paragraphs.get(0);
assertEquals(cp.getStatus(), Status.READY);
@ -437,28 +441,28 @@ public class NotebookTest implements JobListenerFactory{
Set<String> owners = new HashSet<>();
owners.add("user1");
assertEquals(owners, notebook.getNotebookAuthorization().getOwners(cloneNote2.getId()));
notebook.removeNote(note.getId(), null);
notebook.removeNote(cloneNote.getId(), null);
notebook.removeNote(cloneNote2.getId(), null);
notebook.removeNote(note.getId(), anonymous);
notebook.removeNote(cloneNote.getId(), anonymous);
notebook.removeNote(cloneNote2.getId(), anonymous);
}
@Test
public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
Note cloneNote = notebook.cloneNote(note.getId(), null, null);
Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous);
assertEquals(cloneNote.getName(), "Note " + cloneNote.getId());
notebook.removeNote(note.getId(), null);
notebook.removeNote(cloneNote.getId(), null);
notebook.removeNote(note.getId(), anonymous);
notebook.removeNote(cloneNote.getId(), anonymous);
}
@Test
public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
final Paragraph p = note.addParagraph();
p.setText("hello world");
@ -469,21 +473,21 @@ public class NotebookTest implements JobListenerFactory{
// Force paragraph to have String type object
p.setResult("Exception");
Note cloneNote = notebook.cloneNote(note.getId(), "clone note with Exception result", null);
Note cloneNote = notebook.cloneNote(note.getId(), "clone note with Exception result", anonymous);
Paragraph cp = cloneNote.paragraphs.get(0);
// Keep same ParagraphID
assertEquals(cp.getId(), p.getId());
assertEquals(cp.text, p.text);
assertNull(cp.getResult());
notebook.removeNote(note.getId(), null);
notebook.removeNote(cloneNote.getId(), null);
notebook.removeNote(note.getId(), anonymous);
notebook.removeNote(cloneNote.getId(), anonymous);
}
@Test
public void testResourceRemovealOnParagraphNoteRemove() throws IOException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
}
@ -499,11 +503,11 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(2, ResourcePoolUtils.getAllResources().size());
// remove a paragraph
note.removeParagraph(p1.getId());
note.removeParagraph(anonymous.getUser(), p1.getId());
assertEquals(1, ResourcePoolUtils.getAllResources().size());
// remove note
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
assertEquals(0, ResourcePoolUtils.getAllResources().size());
}
@ -511,11 +515,11 @@ public class NotebookTest implements JobListenerFactory{
public void testAngularObjectRemovalOnNotebookRemove() throws InterruptedException,
IOException {
// create a note and a paragraph
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
AngularObjectRegistry registry = factory
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addParagraph();
@ -530,7 +534,7 @@ public class NotebookTest implements JobListenerFactory{
registry.add("o3", "object3", null, null);
// remove notebook
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
// notebook scope or paragraph scope object should be removed
assertNull(registry.get("o1", note.getId(), null));
@ -544,11 +548,11 @@ public class NotebookTest implements JobListenerFactory{
public void testAngularObjectRemovalOnParagraphRemove() throws InterruptedException,
IOException {
// create a note and a paragraph
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
AngularObjectRegistry registry = factory
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addParagraph();
@ -563,7 +567,7 @@ public class NotebookTest implements JobListenerFactory{
registry.add("o3", "object3", null, null);
// remove notebook
note.removeParagraph(p1.getId());
note.removeParagraph(anonymous.getUser(), p1.getId());
// paragraph scope should be removed
assertNull(registry.get("o1", note.getId(), null));
@ -571,18 +575,18 @@ public class NotebookTest implements JobListenerFactory{
// notebook scope and global object sould be remained
assertNotNull(registry.get("o2", note.getId(), null));
assertNotNull(registry.get("o3", null, null));
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedException,
IOException {
// create a note and a paragraph
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
AngularObjectRegistry registry = factory
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// add local scope object
@ -592,19 +596,19 @@ public class NotebookTest implements JobListenerFactory{
// restart interpreter
factory.restart(factory.getInterpreterSettings(note.getId()).get(0).getId());
registry = factory.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess")
registry = factory.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// local and global scope object should be removed
assertNull(registry.get("o1", note.getId(), null));
assertNull(registry.get("o2", null, null));
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testPermissions() throws IOException {
// create a note and a paragraph
Note note = notebook.createNote(null);
Note note = notebook.createNote(anonymous);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
// empty owners, readers or writers means note is public
assertEquals(notebookAuthorization.isOwner(note.getId(),
@ -643,14 +647,14 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(notebookAuthorization.isReader(note.getId(),
new HashSet<String>(Arrays.asList("user3"))), true);
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException,
IOException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
Note note = notebook.createNote(anonymous);
factory.setInterpreters(anonymous.getUser(), note.getId(), factory.getDefaultInterpreterSettingList());
ArrayList<Paragraph> paragraphs = new ArrayList<>();
for (int i = 0; i < 100; i++) {
@ -681,19 +685,20 @@ public class NotebookTest implements JobListenerFactory{
}
assertTrue(isAborted);
notebook.removeNote(note.getId(), null);
notebook.removeNote(note.getId(), anonymous);
}
@Test
public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException {
// create a notes
Note note1 = notebook.createNote(null);
Note note1 = notebook.createNote(anonymous);
Paragraph p1 = note1.addParagraph();
p1.setText("getId");
p1.setAuthenticationInfo(anonymous);
// restart interpreter with per note session enabled
// restart interpreter with per user session enabled
for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNoteSession(true);
setting.getOption().setPerNote(setting.getOption().SCOPED);
notebook.getInterpreterFactory().restart(setting.getId());
}
@ -702,31 +707,34 @@ public class NotebookTest implements JobListenerFactory{
InterpreterResult result = p1.getResult();
// remove note and recreate
notebook.removeNote(note1.getId(), null);
note1 = notebook.createNote(null);
notebook.removeNote(note1.getId(), anonymous);
note1 = notebook.createNote(anonymous);
p1 = note1.addParagraph();
p1.setText("getId");
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
while (p1.getStatus() != Status.FINISHED) Thread.yield();
assertNotEquals(p1.getResult().message(), result.message());
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@Test
public void testInterpreterForNote() throws IOException {
public void testPerSessionInterpreter() throws IOException {
// create two notes
Note note1 = notebook.createNote(null);
Note note1 = notebook.createNote(anonymous);
Paragraph p1 = note1.addParagraph();
Note note2 = notebook.createNote(null);
Note note2 = notebook.createNote(anonymous);
Paragraph p2 = note2.addParagraph();
p1.setText("getId");
p1.setAuthenticationInfo(anonymous);
p2.setText("getId");
p2.setAuthenticationInfo(anonymous);
// run per note session disabled (shared mode)
// run per note session disabled
note1.run(p1.getId());
note2.run(p2.getId());
@ -736,10 +744,9 @@ public class NotebookTest implements JobListenerFactory{
assertEquals(p1.getResult().message(), p2.getResult().message());
// restart interpreter with scoped(per note session) mode enabled
// restart interpreter with per note session enabled
for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNoteSession(true);
setting.getOption().setPerNoteProcess(false);
setting.getOption().setPerNote(InterpreterOption.SCOPED);
notebook.getInterpreterFactory().restart(setting.getId());
}
@ -752,12 +759,55 @@ public class NotebookTest implements JobListenerFactory{
assertNotEquals(p1.getResult().message(), p2.getResult().message());
notebook.removeNote(note1.getId(), anonymous);
notebook.removeNote(note2.getId(), anonymous);
}
// restart interpreter with isolated(per note process) mode enabled
@Test
public void testPerNoteSessionInterpreter() throws IOException {
// create two notes
Note note1 = notebook.createNote(anonymous);
Paragraph p1 = note1.addParagraph();
Note note2 = notebook.createNote(anonymous);
Paragraph p2 = note2.addParagraph();
p1.setText("getId");
p1.setAuthenticationInfo(anonymous);
p2.setText("getId");
p2.setAuthenticationInfo(anonymous);
// shared mode.
note1.run(p1.getId());
note2.run(p2.getId());
while (p1.getStatus() != Status.FINISHED) Thread.yield();
while (p2.getStatus() != Status.FINISHED) Thread.yield();
assertEquals(p1.getResult().message(), p2.getResult().message());
// restart interpreter with scoped mode enabled
for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNoteSession(false);
setting.getOption().setPerNoteProcess(true);
notebook.getInterpreterFactory().restart(setting.getId());
setting.getOption().setPerNote(InterpreterOption.SCOPED);
notebook.getInterpreterFactory().restart(setting.getId(), note1.getId());
notebook.getInterpreterFactory().restart(setting.getId(), note2.getId());
}
// run per note session enabled
note1.run(p1.getId());
note2.run(p2.getId());
while (p1.getStatus() != Status.FINISHED) Thread.yield();
while (p2.getStatus() != Status.FINISHED) Thread.yield();
assertNotEquals(p1.getResult().message(), p2.getResult().message());
// restart interpreter with isolated mode enabled
for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNote(InterpreterOption.ISOLATED);
notebook.getInterpreterFactory().restart(setting.getId(), note1.getId());
notebook.getInterpreterFactory().restart(setting.getId(), note2.getId());
}
// run per note process enabled
@ -769,20 +819,21 @@ public class NotebookTest implements JobListenerFactory{
assertNotEquals(p1.getResult().message(), p2.getResult().message());
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note2.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
notebook.removeNote(note2.getId(), anonymous);
}
@Test
public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IOException {
// create a notes
Note note1 = notebook.createNote(null);
Note note1 = notebook.createNote(anonymous);
Paragraph p1 = note1.addParagraph();
p1.setAuthenticationInfo(anonymous);
p1.setText("getId");
// restart interpreter with per note session enabled
for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) {
setting.getOption().setPerNoteSession(true);
setting.getOption().setPerNote(InterpreterOption.SCOPED);
notebook.getInterpreterFactory().restart(setting.getId());
}
@ -793,15 +844,15 @@ public class NotebookTest implements JobListenerFactory{
// unbind, and rebind setting. that result interpreter instance close
List<String> bindedSettings = notebook.getBindedInterpreterSettingsIds(note1.getId());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
notebook.bindInterpretersToNote(note1.getId(), bindedSettings);
notebook.bindInterpretersToNote(anonymous.getUser(), note1.getId(), new LinkedList<String>());
notebook.bindInterpretersToNote(anonymous.getUser(), note1.getId(), bindedSettings);
note1.run(p1.getId());
while (p1.getStatus() != Status.FINISHED) Thread.yield();
assertNotEquals(result.message(), p1.getResult().message());
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@Test
@ -843,7 +894,7 @@ public class NotebookTest implements JobListenerFactory{
}
});
Note note1 = notebook.createNote(null);
Note note1 = notebook.createNote(anonymous);
assertEquals(1, onNoteCreate.get());
Paragraph p1 = note1.addParagraph();
@ -852,14 +903,14 @@ public class NotebookTest implements JobListenerFactory{
note1.addCloneParagraph(p1);
assertEquals(2, onParagraphCreate.get());
note1.removeParagraph(p1.getId());
note1.removeParagraph(anonymous.getUser(), p1.getId());
assertEquals(1, onParagraphRemove.get());
List<String> settings = notebook.getBindedInterpreterSettingsIds(note1.getId());
notebook.bindInterpretersToNote(note1.getId(), new LinkedList<String>());
notebook.bindInterpretersToNote(anonymous.getUser(), note1.getId(), new LinkedList<String>());
assertEquals(settings.size(), unbindInterpreter.get());
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
assertEquals(1, onNoteRemove.get());
assertEquals(1, onParagraphRemove.get());
}
@ -867,7 +918,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testNormalizeNoteName() throws IOException {
// create a notes
Note note1 = notebook.createNote(null);
Note note1 = notebook.createNote(anonymous);
note1.setName("MyNote");
assertEquals(note1.getName(), "MyNote");
@ -887,43 +938,43 @@ public class NotebookTest implements JobListenerFactory{
note1.setName("\\\\\\MyNote///sub");
assertEquals(note1.getName(), "/MyNote/sub");
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note1.getId(), anonymous);
}
@Test
public void testGetAllNotes() throws Exception {
Note note1 = notebook.createNote(null);
Note note2 = notebook.createNote(null);
assertEquals(2, notebook.getAllNotes(new AuthenticationInfo("anonymous")).size());
Note note1 = notebook.createNote(anonymous);
Note note2 = notebook.createNote(anonymous);
assertEquals(2, notebook.getAllNotes(Sets.newHashSet("anonymous")).size());
notebook.getNotebookAuthorization().setOwners(note1.getId(), Sets.newHashSet("user1"));
notebook.getNotebookAuthorization().setWriters(note1.getId(), Sets.newHashSet("user1"));
notebook.getNotebookAuthorization().setReaders(note1.getId(), Sets.newHashSet("user1"));
assertEquals(1, notebook.getAllNotes(new AuthenticationInfo("anonymous")).size());
assertEquals(2, notebook.getAllNotes(new AuthenticationInfo("user1")).size());
assertEquals(1, notebook.getAllNotes(Sets.newHashSet("anonymous")).size());
assertEquals(2, notebook.getAllNotes(Sets.newHashSet("user1")).size());
notebook.getNotebookAuthorization().setOwners(note2.getId(), Sets.newHashSet("user2"));
notebook.getNotebookAuthorization().setWriters(note2.getId(), Sets.newHashSet("user2"));
notebook.getNotebookAuthorization().setReaders(note2.getId(), Sets.newHashSet("user2"));
assertEquals(0, notebook.getAllNotes(new AuthenticationInfo("anonymous")).size());
assertEquals(1, notebook.getAllNotes(new AuthenticationInfo("user1")).size());
assertEquals(1, notebook.getAllNotes(new AuthenticationInfo("user2")).size());
notebook.removeNote(note1.getId(), null);
notebook.removeNote(note2.getId(), null);
assertEquals(0, notebook.getAllNotes(Sets.newHashSet("anonymous")).size());
assertEquals(1, notebook.getAllNotes(Sets.newHashSet("user1")).size());
assertEquals(1, notebook.getAllNotes(Sets.newHashSet("user2")).size());
notebook.removeNote(note1.getId(), anonymous);
notebook.removeNote(note2.getId(), anonymous);
}
@Test
public void testGetAllNotesWithDifferentPermissions() throws IOException {
AuthenticationInfo user1 = new AuthenticationInfo("user1");
AuthenticationInfo user2 = new AuthenticationInfo("user2");
HashSet<String> user1 = Sets.newHashSet("user1");
HashSet<String> user2 = Sets.newHashSet("user1");
List<Note> notes1 = notebook.getAllNotes(user1);
List<Note> notes2 = notebook.getAllNotes(user2);
assertEquals(notes1.size(), 0);
assertEquals(notes2.size(), 0);
//creates note and sets user1 owner
Note note = notebook.createNote(user1);
Note note = notebook.createNote(new AuthenticationInfo("user1"));
// note is public since readers and writers empty
notes1 = notebook.getAllNotes(user1);
@ -942,7 +993,7 @@ public class NotebookTest implements JobListenerFactory{
notes1 = notebook.getAllNotes(user1);
notes2 = notebook.getAllNotes(user2);
assertEquals(notes1.size(), 1);
assertEquals(notes2.size(), 0);
assertEquals(notes2.size(), 1);
}
private void delete(File file){

93
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
View file

@ -42,6 +42,7 @@ import org.apache.zeppelin.scheduler.JobListener;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.search.LuceneSearch;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
import org.junit.After;
import org.junit.Before;
@ -65,6 +66,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
private SearchService search;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private AuthenticationInfo anonymous;
private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
@Before
@ -95,7 +97,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);
@ -103,6 +105,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this, search,
notebookAuthorization, credentials);
anonymous = new AuthenticationInfo("anonymous");
}
@After
@ -119,40 +122,40 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void testSyncOnCreate() throws IOException {
/* check that both storage systems are empty */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(0, notebookRepoSync.list(1, null).size());
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
/* create note */
Note note = notebookSync.createNote(null);
Note note = notebookSync.createNote(anonymous);
// check that automatically saved on both storages
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
assertEquals(notebookRepoSync.list(0, null).get(0).getId(),notebookRepoSync.list(1, null).get(0).getId());
notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), null);
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous);
}
@Test
public void testSyncOnDelete() throws IOException {
/* create note */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(0, notebookRepoSync.list(1, null).size());
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
Note note = notebookSync.createNote(null);
Note note = notebookSync.createNote(anonymous);
/* check that created in both storage systems */
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
assertEquals(notebookRepoSync.list(0, null).get(0).getId(),notebookRepoSync.list(1, null).get(0).getId());
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
/* remove Note */
notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), null);
notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous);
/* check that deleted in both storages */
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(0, notebookRepoSync.list(1, null).size());
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
}
@ -160,7 +163,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void testSyncUpdateMain() throws IOException {
/* create note */
Note note = notebookSync.createNote(null);
Note note = notebookSync.createNote(anonymous);
Paragraph p1 = note.addParagraph();
Map config = p1.getConfig();
config.put("enabled", true);
@ -172,38 +175,38 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
/* new paragraph not yet saved into storages */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
/* save to storage under index 0 (first storage) */
notebookRepoSync.save(0, note, null);
notebookRepoSync.save(0, note, anonymous);
/* check paragraph saved to first storage */
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
/* check paragraph isn't saved to second storage */
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
/* apply sync */
notebookRepoSync.sync(null);
/* check whether added to second storage */
assertEquals(1, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
/* check whether same paragraph id */
assertEquals(p1.getId(), notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getLastParagraph().getId());
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getLastParagraph().getId());
assertEquals(p1.getId(), notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getLastParagraph().getId());
notebookRepoSync.remove(note.getId(), null);
notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getLastParagraph().getId());
notebookRepoSync.remove(note.getId(), anonymous);
}
@Test
public void testSyncOnReloadedList() throws IOException {
/* check that both storage repos are empty */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(0, notebookRepoSync.list(1, null).size());
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
File srcDir = new File("src/test/resources/2A94M5J1Z");
File destDir = new File(secNotebookDir + "/2A94M5J1Z");
@ -214,13 +217,13 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
} catch (IOException e) {
LOG.error(e.toString(), e);
}
assertEquals(0, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
// After reloading notebooks repos should be synchronized
notebookSync.reloadAllNotes(null);
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
notebookSync.reloadAllNotes(anonymous);
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
}
@Test
@ -287,15 +290,15 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0);
// no notes
assertThat(vRepoSync.list(null).size()).isEqualTo(0);
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(0);
// create note
Note note = vNotebookSync.createNote(null);
assertThat(vRepoSync.list(null).size()).isEqualTo(1);
Note note = vNotebookSync.createNote(anonymous);
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(1);
String noteId = vRepoSync.list(null).get(0).getId();
String noteId = vRepoSync.list(anonymous).get(0).getId();
// first checkpoint
vRepoSync.checkpoint(noteId, "checkpoint message", null);
int vCount = gitRepo.revisionHistory(noteId, null).size();
vRepoSync.checkpoint(noteId, "checkpoint message", anonymous);
int vCount = gitRepo.revisionHistory(noteId, anonymous).size();
assertThat(vCount).isEqualTo(1);
Paragraph p = note.addParagraph();
@ -305,10 +308,10 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
p.setText("%md checkpoint test");
// save and checkpoint again
vRepoSync.save(note, null);
vRepoSync.checkpoint(noteId, "checkpoint message 2", null);
assertThat(gitRepo.revisionHistory(noteId, null).size()).isEqualTo(vCount + 1);
notebookRepoSync.remove(note.getId(), null);
vRepoSync.save(note, anonymous);
vRepoSync.checkpoint(noteId, "checkpoint message 2", anonymous);
assertThat(gitRepo.revisionHistory(noteId, anonymous).size()).isEqualTo(vCount + 1);
notebookRepoSync.remove(note.getId(), anonymous);
}
static void delete(File file){

10
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
View file

@ -32,10 +32,12 @@ import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.notebook.repo.zeppelinhub.security.Authentication;
import org.apache.zeppelin.scheduler.JobListener;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.search.LuceneSearch;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -76,7 +78,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
this.schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
@ -106,14 +108,16 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
@Test
public void testSaveNotebook() throws IOException, InterruptedException {
Note note = notebook.createNote(null);
factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList());
AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
Note note = notebook.createNote(anonymous);
factory.setInterpreters("user", note.getId(), factory.getDefaultInterpreterSettingList());
Paragraph p1 = note.addParagraph();
Map<String, Object> config = p1.getConfig();
config.put("enabled", true);
p1.setConfig(config);
p1.setText("%mock1 hello world");
p1.setAuthenticationInfo(anonymous);
note.run(p1.getId());
int timeout = 1;

7
zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java
View file

@ -29,6 +29,7 @@ import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@ -41,6 +42,7 @@ public class LuceneSearchTest {
private static NotebookRepo notebookRepoMock;
private static InterpreterFactory interpreterFactory;
private SearchService notebookIndex;
private AuthenticationInfo anonymous;
@BeforeClass
public static void beforeStartUp() {
@ -54,6 +56,7 @@ public class LuceneSearchTest {
@Before
public void startUp() {
notebookIndex = new LuceneSearch();
anonymous = new AuthenticationInfo("anonymous");
}
@After
@ -202,7 +205,7 @@ public class LuceneSearchTest {
//when
Paragraph p1 = note1.getLastParagraph();
p1.setText("no no no");
note1.persist(null);
note1.persist(anonymous);
//then
assertThat(resultForQuery("Notebook1").size()).isEqualTo(1);
@ -226,7 +229,7 @@ public class LuceneSearchTest {
//when
note1.setName("NotebookN");
note1.persist(null);
note1.persist(anonymous);
//then
assertThat(resultForQuery("Notebook1")).isEmpty();