Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[ZEPPELIN-2627] Interpreter Component Refactoring

Browse source
This commit is contained in:
Jeff Zhang 2017-05-23 15:52:15 +08:00
parent 69d58f373b
commit 74bcb9119a
90 changed files with 4215 additions and 5384 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.travis.yml
View file

@ -162,6 +162,7 @@ after_success:
after_failure:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
- find . -name rat.txt | xargs cat
- cat logs/*
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log

8
helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
View file

@ -43,13 +43,13 @@ public class ZeppelinDevServer extends
}
@Override
protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
protected Interpreter getInterpreter(String sessionId, String className) throws TException {
synchronized (this) {
InterpreterGroup interpreterGroup = getInterpreterGroup();
if (interpreterGroup == null || interpreterGroup.isEmpty()) {
createInterpreter(
"dev",
sessionKey,
sessionId,
DevInterpreter.class.getName(),
new HashMap<String, String>(),
"anonymous");
@ -57,11 +57,11 @@ public class ZeppelinDevServer extends
}
}
Interpreter intp = super.getInterpreter(sessionKey, className);
Interpreter intp = super.getInterpreter(sessionId, className);
interpreter = (DevInterpreter) (
((LazyOpenInterpreter) intp).getInnerInterpreter());
interpreter.setInterpreterEvent(this);
return super.getInterpreter(sessionKey, className);
return super.getInterpreter(sessionId, className);
}
@Override

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -149,7 +149,6 @@ public abstract class Interpreter {
@ZeppelinApi
public Interpreter(Properties property) {
logger.debug("Properties: {}", property);
this.property = property;
}

239
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -17,107 +17,90 @@
package org.apache.zeppelin.interpreter;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
/**
* InterpreterGroup is list of interpreters in the same interpreter group.
* For example spark, pyspark, sql interpreters are in the same 'spark' group
* and InterpreterGroup will have reference to these all interpreters.
* InterpreterGroup is collections of interpreter sessions.
* One session could include multiple interpreters.
* For example spark, pyspark, sql interpreters are in the same 'spark' interpreter session.
*
* Remember, list of interpreters are dedicated to a session. Session could be shared across user
* or notes, so the sessionId could be user or noteId or their combination.
* So InterpreterGroup internally manages map of [interpreterSessionKey(noteId, user, or
* So InterpreterGroup internally manages map of [sessionId(noteId, user, or
* their combination), list of interpreters]
*
* A InterpreterGroup runs on interpreter process.
* And unit of interpreter instantiate, restart, bind, unbind.
* A InterpreterGroup runs interpreter process while its subclass ManagedInterpreterGroup runs
* in zeppelin server process.
*/
public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
String id;
public class InterpreterGroup {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroup.class);
AngularObjectRegistry angularObjectRegistry;
InterpreterHookRegistry hookRegistry;
RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
ResourcePool resourcePool;
boolean angularRegistryPushed = false;
// map [notebook session, Interpreters in the group], to support per note session interpreters
//Map<String, List<Interpreter>> interpreters = new ConcurrentHashMap<String,
// List<Interpreter>>();
private static final Map<String, InterpreterGroup> allInterpreterGroups =
new ConcurrentHashMap<>();
public static InterpreterGroup getByInterpreterGroupId(String id) {
return allInterpreterGroups.get(id);
}
public static Collection<InterpreterGroup> getAll() {
return new LinkedList(allInterpreterGroups.values());
}
protected String id;
// sessionId --> interpreters
protected Map<String, List<Interpreter>> sessions = new ConcurrentHashMap();
private AngularObjectRegistry angularObjectRegistry;
private InterpreterHookRegistry hookRegistry;
private ResourcePool resourcePool;
private boolean angularRegistryPushed = false;
/**
* Create InterpreterGroup with given id
* Create InterpreterGroup with given id, used in InterpreterProcess
* @param id
*/
public InterpreterGroup(String id) {
this.id = id;
allInterpreterGroups.put(id, this);
}
/**
* Create InterpreterGroup with autogenerated id
*/
public InterpreterGroup() {
getId();
allInterpreterGroups.put(id, this);
this.id = generateId();
}
private static String generateId() {
return "InterpreterGroup_" + System.currentTimeMillis() + "_"
+ new Random().nextInt();
return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new Random().nextInt();
}
public String getId() {
synchronized (this) {
if (id == null) {
id = generateId();
}
return id;
}
return this.id;
}
/**
* Get combined property of all interpreters in this group
* @return
*/
public Properties getProperty() {
Properties p = new Properties();
//TODO(zjffdu) change it to getSession. For now just keep this method to reduce code change
public synchronized List<Interpreter> get(String sessionId) {
return sessions.get(sessionId);
}
for (List<Interpreter> intpGroupForASession : this.values()) {
for (Interpreter intp : intpGroupForASession) {
p.putAll(intp.getProperty());
}
// it's okay to break here while every List<Interpreters> will have the same property set
break;
//TODO(zjffdu) change it to addSession. For now just keep this method to reduce code change
public synchronized void put(String sessionId, List<Interpreter> interpreters) {
this.sessions.put(sessionId, interpreters);
}
public synchronized void addInterpreterToSession(Interpreter interpreter, String sessionId) {
LOGGER.debug("Add Interpreter {} to session {}", interpreter.getClassName(), sessionId);
List<Interpreter> interpreters = get(sessionId);
if (interpreters == null) {
interpreters = new ArrayList<>();
}
return p;
interpreters.add(interpreter);
put(sessionId, interpreters);
}
//TODO(zjffdu) rename it to a more proper name.
//For now just keep this method to reduce code change
public Collection<List<Interpreter>> values() {
return sessions.values();
}
public AngularObjectRegistry getAngularObjectRegistry() {
@ -136,128 +119,8 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
this.hookRegistry = hookRegistry;
}
public RemoteInterpreterProcess getRemoteInterpreterProcess() {
return remoteInterpreterProcess;
}
public void setRemoteInterpreterProcess(RemoteInterpreterProcess remoteInterpreterProcess) {
this.remoteInterpreterProcess = remoteInterpreterProcess;
}
/**
* Close all interpreter instances in this group
*/
public void close() {
LOGGER.info("Close interpreter group " + getId());
List<Interpreter> intpToClose = new LinkedList<>();
for (List<Interpreter> intpGroupForSession : this.values()) {
intpToClose.addAll(intpGroupForSession);
}
close(intpToClose);
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);
}
/**
* Close all interpreter instances in this group for the session
* @param sessionId
*/
public void close(String sessionId) {
LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
final List<Interpreter> intpForSession = this.get(sessionId);
close(intpForSession);
}
private void close(final Collection<Interpreter> intpToClose) {
close(null, null, null, intpToClose);
}
public void close(final Map<String, InterpreterGroup> interpreterGroupRef,
final String processKey, final String sessionKey) {
LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionKey);
close(interpreterGroupRef, processKey, sessionKey, this.get(sessionKey));
}
private void close(final Map<String, InterpreterGroup> interpreterGroupRef,
final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
if (intpToClose == null) {
return;
}
Thread t = new Thread() {
public void run() {
for (Interpreter interpreter : intpToClose) {
Scheduler scheduler = interpreter.getScheduler();
interpreter.close();
if (null != scheduler) {
SchedulerFactory.singleton().removeScheduler(scheduler.getName());
}
}
if (remoteInterpreterProcess != null) {
//TODO(jl): Because interpreter.close() runs as a seprate thread, we cannot guarantee
// refernceCount is a proper value. And as the same reason, we must not call
// remoteInterpreterProcess.dereference twice - this method also be called by
// interpreter.close().
// remoteInterpreterProcess.dereference();
if (remoteInterpreterProcess.referenceCount() <= 0) {
remoteInterpreterProcess = null;
allInterpreterGroups.remove(id);
}
}
// TODO(jl): While closing interpreters in a same session, we should remove after all
// interpreters are removed. OMG. It's too dirty!!
if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
interpreterGroupRef.remove(processKey);
} else {
interpreterGroup.remove(sessionKey);
}
}
}
};
t.start();
try {
t.join();
} catch (InterruptedException e) {
LOGGER.error("Can't close interpreter: {}", getId(), e);
}
}
/**
* Close all interpreter instances in this group
*/
public void shutdown() {
LOGGER.info("Close interpreter group " + getId());
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);
List<Interpreter> intpToClose = new LinkedList<>();
for (List<Interpreter> intpGroupForSession : this.values()) {
intpToClose.addAll(intpGroupForSession);
}
close(intpToClose);
public int getSessionNum() {
return sessions.size();
}
public void setResourcePool(ResourcePool resourcePool) {
@ -275,4 +138,8 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
public void setAngularRegistryPushed(boolean angularRegistryPushed) {
this.angularRegistryPushed = angularRegistryPushed;
}
public boolean isEmpty() {
return sessions.isEmpty();
}
}

9
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
View file

@ -12,6 +12,15 @@ public class InterpreterRunner {
@SerializedName("win")
private String winPath;
public InterpreterRunner() {
}
public InterpreterRunner(String linuxPath, String winPath) {
this.linuxPath = linuxPath;
this.winPath = winPath;
}
public String getPath() {
return System.getProperty("os.name").startsWith("Windows") ? winPath : linuxPath;
}

64
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -106,9 +106,14 @@ public class RemoteInterpreterServer
@Override
public void shutdown() throws TException {
logger.info("Shutting down...");
eventClient.waitForEventQueueBecomesEmpty(DEFAULT_SHUTDOWN_TIMEOUT);
if (interpreterGroup != null) {
interpreterGroup.close();
for (List<Interpreter> session : interpreterGroup.values()) {
for (Interpreter interpreter : session) {
interpreter.close();
}
}
}
server.stop();
@ -159,7 +164,7 @@ public class RemoteInterpreterServer
}
@Override
public void createInterpreter(String interpreterGroupId, String sessionKey, String
public void createInterpreter(String interpreterGroupId, String sessionId, String
className, Map<String, String> properties, String userName) throws TException {
if (interpreterGroup == null) {
interpreterGroup = new InterpreterGroup(interpreterGroupId);
@ -190,20 +195,11 @@ public class RemoteInterpreterServer
replClass.getConstructor(new Class[] {Properties.class});
Interpreter repl = constructor.newInstance(p);
repl.setClassloaderUrls(new URL[]{});
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
if (interpreters == null) {
interpreters = new LinkedList<>();
interpreterGroup.put(sessionKey, interpreters);
}
interpreters.add(new LazyOpenInterpreter(repl));
}
logger.info("Instantiate interpreter {}", className);
repl.setInterpreterGroup(interpreterGroup);
repl.setUserName(userName);
interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(repl), sessionId);
} catch (ClassNotFoundException | NoSuchMethodException | SecurityException
| InstantiationException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
@ -237,13 +233,13 @@ public class RemoteInterpreterServer
}
}
protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
protected Interpreter getInterpreter(String sessionId, String className) throws TException {
if (interpreterGroup == null) {
throw new TException(
new InterpreterException("Interpreter instance " + className + " not created"));
}
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
List<Interpreter> interpreters = interpreterGroup.get(sessionId);
if (interpreters == null) {
throw new TException(
new InterpreterException("Interpreter " + className + " not initialized"));
@ -259,19 +255,20 @@ public class RemoteInterpreterServer
}
@Override
public void open(String noteId, String className) throws TException {
Interpreter intp = getInterpreter(noteId, className);
public void open(String sessionId, String className) throws TException {
logger.info(String.format("Open Interpreter %s for session %s ", className, sessionId));
Interpreter intp = getInterpreter(sessionId, className);
intp.open();
}
@Override
public void close(String sessionKey, String className) throws TException {
public void close(String sessionId, String className) throws TException {
// unload all applications
for (String appId : runningApplications.keySet()) {
RunningApplication appInfo = runningApplications.get(appId);
// see NoteInterpreterLoader.SHARED_SESSION
if (appInfo.noteId.equals(sessionKey) || sessionKey.equals("shared_session")) {
if (appInfo.noteId.equals(sessionId) || sessionId.equals("shared_session")) {
try {
logger.info("Unload App {} ", appInfo.pkg.getName());
appInfo.app.unload();
@ -286,7 +283,7 @@ public class RemoteInterpreterServer
// close interpreters
List<Interpreter> interpreters;
synchronized (interpreterGroup) {
interpreters = interpreterGroup.get(sessionKey);
interpreters = interpreterGroup.get(sessionId);
}
if (interpreters != null) {
Iterator<Interpreter> it = interpreters.iterator();
@ -322,7 +319,6 @@ public class RemoteInterpreterServer
intp,
st,
context);
scheduler.submit(job);
while (!job.isTerminated()) {
@ -566,30 +562,34 @@ public class RemoteInterpreterServer
}
@Override
public int getProgress(String noteId, String className,
public int getProgress(String sessionId, String className,
RemoteInterpreterContext interpreterContext)
throws TException {
Integer manuallyProvidedProgress = progressMap.get(interpreterContext.getParagraphId());
if (manuallyProvidedProgress != null) {
return manuallyProvidedProgress;
} else {
Interpreter intp = getInterpreter(noteId, className);
Interpreter intp = getInterpreter(sessionId, className);
if (intp == null) {
throw new TException("No interpreter {} existed for session {}".format(
className, sessionId));
}
return intp.getProgress(convert(interpreterContext, null));
}
}
@Override
public String getFormType(String noteId, String className) throws TException {
Interpreter intp = getInterpreter(noteId, className);
public String getFormType(String sessionId, String className) throws TException {
Interpreter intp = getInterpreter(sessionId, className);
return intp.getFormType().toString();
}
@Override
public List<InterpreterCompletion> completion(String noteId,
public List<InterpreterCompletion> completion(String sessionId,
String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext)
throws TException {
Interpreter intp = getInterpreter(noteId, className);
Interpreter intp = getInterpreter(sessionId, className);
List completion = intp.completion(buf, cursor, convert(remoteInterpreterContext, null));
return completion;
}
@ -766,16 +766,16 @@ public class RemoteInterpreterServer
}
@Override
public String getStatus(String sessionKey, String jobId)
public String getStatus(String sessionId, String jobId)
throws TException {
if (interpreterGroup == null) {
return "Unknown";
return Status.UNKNOWN.name();
}
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
List<Interpreter> interpreters = interpreterGroup.get(sessionId);
if (interpreters == null) {
return "Unknown";
return Status.UNKNOWN.name();
}
for (Interpreter intp : interpreters) {
@ -792,7 +792,7 @@ public class RemoteInterpreterServer
}
}
}
return "Unknown";
return Status.UNKNOWN.name();
}

138
zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java
View file

@ -1,138 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.resource;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.slf4j.Logger;
import java.util.List;
/**
* Utilities for ResourcePool
*/
public class ResourcePoolUtils {
static Logger logger = org.slf4j.LoggerFactory.getLogger(ResourcePoolUtils.class);
public static ResourceSet getAllResources() {
return getAllResourcesExcept(null);
}
public static ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
if (interpreterGroupExcludsion != null &&
intpGroup.getId().equals(interpreterGroupExcludsion)) {
continue;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
} else if (remoteInterpreterProcess.isRunning()) {
RemoteInterpreterService.Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
if (client == null) {
// remote interpreter may not started yet or terminated.
continue;
}
List<String> resourceList = client.resourcePoolGetAll();
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
}
}
return resourceSet;
}
public static void removeResourcesBelongsToNote(String noteId) {
removeResourcesBelongsToParagraph(noteId, null);
}
public static void removeResourcesBelongsToParagraph(String noteId, String paragraphId) {
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
ResourceSet resourceSet = new ResourceSet();
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
if (noteId != null) {
resourceSet = resourceSet.filterByNoteId(noteId);
}
if (paragraphId != null) {
resourceSet = resourceSet.filterByParagraphId(paragraphId);
}
for (Resource r : resourceSet) {
localPool.remove(
r.getResourceId().getNoteId(),
r.getResourceId().getParagraphId(),
r.getResourceId().getName());
}
} else if (remoteInterpreterProcess.isRunning()) {
RemoteInterpreterService.Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
List<String> resourceList = client.resourcePoolGetAll();
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
if (noteId != null) {
resourceSet = resourceSet.filterByNoteId(noteId);
}
if (paragraphId != null) {
resourceSet = resourceSet.filterByParagraphId(paragraphId);
}
for (Resource r : resourceSet) {
client.resourceRemove(
r.getResourceId().getNoteId(),
r.getResourceId().getParagraphId(),
r.getResourceId().getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
}
}
}
}

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
View file

@ -41,6 +41,7 @@ public abstract class Job {
/**
* Job status.
*
* UNKNOWN - Job is not found in remote
* READY - Job is not running, ready to run.
* PENDING - Job is submitted to scheduler. but not running yet
* RUNNING - Job is running.
@ -48,8 +49,8 @@ public abstract class Job {
* ERROR - Job finished run. with error
* ABORT - Job finished by abort
*/
public static enum Status {
READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
public enum Status {
UNKNOWN, READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
public boolean isReady() {
return this == READY;
@ -70,14 +71,14 @@ public abstract class Job {
Date dateCreated;
Date dateStarted;
Date dateFinished;
Status status;
volatile Status status;
static Logger LOGGER = LoggerFactory.getLogger(Job.class);
transient boolean aborted = false;
private String errorMessage;
private transient Throwable exception;
private volatile String errorMessage;
private transient volatile Throwable exception;
private transient JobListener listener;
private long progressUpdateIntervalMs;

63
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
View file

@ -17,24 +17,21 @@
package org.apache.zeppelin.scheduler;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* TODO(moon) : add description.
* Factory class for creating schedulers
*
*/
public class SchedulerFactory implements SchedulerListener {
private static final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
ExecutorService executor;
Map<String, Scheduler> schedulers = new LinkedHashMap<>();
protected ExecutorService executor;
protected Map<String, Scheduler> schedulers = new LinkedHashMap<>();
private static SchedulerFactory singleton;
private static Long singletonLock = new Long(0);
@ -54,17 +51,17 @@ public class SchedulerFactory implements SchedulerListener {
return singleton;
}
public SchedulerFactory() throws Exception {
executor = ExecutorFactory.singleton().createOrGet("schedulerFactory", 100);
SchedulerFactory() throws Exception {
executor = ExecutorFactory.singleton().createOrGet("SchedulerFactory", 100);
}
public void destroy() {
ExecutorFactory.singleton().shutdown("schedulerFactory");
ExecutorFactory.singleton().shutdown("SchedulerFactory");
}
public Scheduler createOrGetFIFOScheduler(String name) {
synchronized (schedulers) {
if (schedulers.containsKey(name) == false) {
if (!schedulers.containsKey(name)) {
Scheduler s = new FIFOScheduler(name, executor, this);
schedulers.put(name, s);
executor.execute(s);
@ -75,7 +72,7 @@ public class SchedulerFactory implements SchedulerListener {
public Scheduler createOrGetParallelScheduler(String name, int maxConcurrency) {
synchronized (schedulers) {
if (schedulers.containsKey(name) == false) {
if (!schedulers.containsKey(name)) {
Scheduler s = new ParallelScheduler(name, executor, this, maxConcurrency);
schedulers.put(name, s);
executor.execute(s);
@ -84,60 +81,38 @@ public class SchedulerFactory implements SchedulerListener {
}
}
public Scheduler createOrGetRemoteScheduler(
String name,
String noteId,
RemoteInterpreterProcess interpreterProcess,
int maxConcurrency) {
public Scheduler createOrGetScheduler(Scheduler scheduler) {
synchronized (schedulers) {
if (schedulers.containsKey(name) == false) {
Scheduler s = new RemoteScheduler(
name,
executor,
noteId,
interpreterProcess,
this,
maxConcurrency);
schedulers.put(name, s);
executor.execute(s);
if (!schedulers.containsKey(scheduler.getName())) {
schedulers.put(scheduler.getName(), scheduler);
executor.execute(scheduler);
}
return schedulers.get(name);
return schedulers.get(scheduler.getName());
}
}
public Scheduler removeScheduler(String name) {
public void removeScheduler(String name) {
synchronized (schedulers) {
Scheduler s = schedulers.remove(name);
if (s != null) {
s.stop();
}
}
return null;
}
public Collection<Scheduler> listScheduler(String name) {
List<Scheduler> s = new LinkedList<>();
synchronized (schedulers) {
for (Scheduler ss : schedulers.values()) {
s.add(ss);
}
}
return s;
public ExecutorService getExecutor() {
return executor;
}
@Override
public void jobStarted(Scheduler scheduler, Job job) {
logger.info("Job " + job.getJobName() + " started by scheduler " + scheduler.getName());
logger.info("Job " + job.getId() + " started by scheduler " + scheduler.getName());
}
@Override
public void jobFinished(Scheduler scheduler, Job job) {
logger.info("Job " + job.getJobName() + " finished by scheduler " + scheduler.getName());
logger.info("Job " + job.getId() + " finished by scheduler " + scheduler.getName());
}
}

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
View file

@ -17,7 +17,6 @@
package org.apache.zeppelin.tabledata;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import java.util.Iterator;

76
zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java Normal file
View file

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.util;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
/**
* Generate Tiny ID.
*/
public class IdHashes {
private static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
'W', 'X', 'Y', 'Z'};
/**
* encodes the given string into the base of the dictionary provided in the constructor.
*
* @param value the number to encode.
* @return the encoded string.
*/
private static String encode(Long value) {
List<Character> result = new ArrayList<>();
BigInteger base = new BigInteger("" + DICTIONARY.length);
int exponent = 1;
BigInteger remaining = new BigInteger(value.toString());
while (true) {
BigInteger a = base.pow(exponent); // 16^1 = 16
BigInteger b = remaining.mod(a); // 119 % 16 = 7 | 112 % 256 = 112
BigInteger c = base.pow(exponent - 1);
BigInteger d = b.divide(c);
// if d > dictionary.length, we have a problem. but BigInteger doesnt have
// a greater than method :-( hope for the best. theoretically, d is always
// an index of the dictionary!
result.add(DICTIONARY[d.intValue()]);
remaining = remaining.subtract(b); // 119 - 7 = 112 | 112 - 112 = 0
// finished?
if (remaining.equals(BigInteger.ZERO)) {
break;
}
exponent++;
}
// need to reverse it, since the start of the list contains the least significant values
StringBuffer sb = new StringBuffer();
for (int i = result.size() - 1; i >= 0; i--) {
sb.append(result.get(i));
}
return sb.toString();
}
public static String generateId() {
return encode(System.currentTimeMillis() + new Random().nextInt());
}
}

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java
View file

@ -17,7 +17,7 @@
package org.apache.zeppelin.util;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.util.Properties;

43
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java
View file

@ -1,43 +0,0 @@
package org.apache.zeppelin.interpreter;
import java.util.Properties;
/**
*
*/
public class DummyInterpreter extends Interpreter {
public DummyInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return null;
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return null;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}

11
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
View file

@ -21,6 +21,7 @@ import static org.junit.Assert.*;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.After;
import org.junit.Before;
@ -29,7 +30,7 @@ import org.junit.Test;
public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChangeListener {
private File tmpDir;
private File fileChanged;
private int numChanged;
private AtomicInteger numChanged;
private InterpreterOutputChangeWatcher watcher;
@Before
@ -40,7 +41,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
fileChanged = null;
numChanged = 0;
numChanged = new AtomicInteger(0);
}
@After
@ -66,7 +67,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
@Test
public void test() throws IOException, InterruptedException {
assertNull(fileChanged);
assertEquals(0, numChanged);
assertEquals(0, numChanged.get());
Thread.sleep(1000);
// create new file
@ -92,14 +93,14 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
}
assertNotNull(fileChanged);
assertEquals(1, numChanged);
assertEquals(1, numChanged.get());
}
@Override
public void fileChanged(File file) {
fileChanged = file;
numChanged++;
numChanged.incrementAndGet();
synchronized(this) {
notify();

38
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
View file

@ -24,6 +24,7 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
//TODO(zjffdu) add more test for Interpreter which is a very important class
public class InterpreterTest {
@Test
@ -85,4 +86,41 @@ public class InterpreterTest {
);
}
public static class DummyInterpreter extends Interpreter {
public DummyInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return null;
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return null;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}
}

5
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
View file

@ -17,12 +17,11 @@
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import java.io.IOException;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
public class RemoteInterpreterUtilsTest {

115
zeppelin-interpreter/src/test/resources/conf/interpreter.json Normal file
View file

@ -0,0 +1,115 @@
{
"interpreterSettings": {
"2C3RWCVAG": {
"id": "2C3RWCVAG",
"name": "test",
"group": "test",
"properties": {
"property_1": "value_1",
"property_2": "new_value_2",
"property_3": "value_3"
},
"status": "READY",
"interpreterGroup": [
{
"name": "echo",
"class": "org.apache.zeppelin.interpreter.EchoInterpreter",
"defaultInterpreter": true,
"editor": {
"language": "java",
"editOnDblClick": false
}
}
],
"dependencies": [],
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
},
"2CKWE7B19": {
"id": "2CKWE7B19",
"name": "test2",
"group": "test",
"properties": {
"property_1": "value_1",
"property_2": "new_value_2",
"property_3": "value_3"
},
"status": "READY",
"interpreterGroup": [
{
"name": "echo",
"class": "org.apache.zeppelin.interpreter.EchoInterpreter",
"defaultInterpreter": true,
"editor": {
"language": "java",
"editOnDblClick": false
}
}
],
"dependencies": [],
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
},
"interpreterBindings": {
"2C6793KRV": [
"2C48Y7FSJ",
"2C63XW4XE",
"2C66GE1VB",
"2C5VH924X",
"2C4BJDRRZ",
"2C3SQSB7V",
"2C4HKDCQW",
"2C3DR183X",
"2C66Z9XPQ",
"2C3PTPMUH",
"2C69WE69N",
"2C5SRRXHM",
"2C4ZD49PF",
"2C6V3D44K",
"2C4UB1UZA",
"2C5S1R21W",
"2C5DCRVGM",
"2C686X8ZH",
"2C3RWCVAG",
"2C3JKFMJU",
"2C3VECEG2"
]
},
"interpreterRepositories": [
{
"id": "central",
"type": "default",
"url": "http://repo1.maven.org/maven2/",
"releasePolicy": {
"enabled": true,
"updatePolicy": "daily",
"checksumPolicy": "warn"
},
"snapshotPolicy": {
"enabled": true,
"updatePolicy": "daily",
"checksumPolicy": "warn"
},
"mirroredRepositories": [],
"repositoryManager": false
}
]
}

42
zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json Normal file
View file

@ -0,0 +1,42 @@
[
{
"group": "test",
"name": "double_echo",
"className": "org.apache.zeppelin.interpreter.DoubleEchoInterpreter",
"properties": {
"property_1": {
"envName": "PROPERTY_1",
"propertyName": "property_1",
"defaultValue": "value_1",
"description": "desc_1"
},
"property_2": {
"envName": "PROPERTY_2",
"propertyName": "property_2",
"defaultValue": "value_2",
"description": "desc_2"
}
}
},
{
"group": "test",
"name": "echo",
"defaultInterpreter": true,
"className": "org.apache.zeppelin.interpreter.EchoInterpreter",
"properties": {
"property_1": {
"envName": "PROPERTY_1",
"propertyName": "property_1",
"defaultValue": "value_1",
"description": "desc_1"
},
"property_2": {
"envName": "PROPERTY_2",
"propertyName": "property_2",
"defaultValue": "value_2",
"description": "desc_2"
}
}
}
]

4
zeppelin-interpreter/src/test/resources/log4j.properties
View file

@ -26,4 +26,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
#
# Root logger option
log4j.rootLogger=INFO, stdout
log4j.rootLogger=INFO, stdout
log4j.logger.org.apache.zeppelin.interpreter=DEBUG
log4j.logger.org.apache.zeppelin.scheduler=DEBUG

4
zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
View file

@ -185,7 +185,7 @@ public class InterpreterRestApi {
String noteId = request == null ? null : request.getNoteId();
if (null == noteId) {
interpreterSettingManager.close(setting);
interpreterSettingManager.close(settingId);
} else {
interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
}
@ -208,7 +208,7 @@ public class InterpreterRestApi {
@GET
@ZeppelinApi
public Response listInterpreter(String message) {
Map<String, InterpreterSetting> m = interpreterSettingManager.getAvailableInterpreterSettings();
Map<String, InterpreterSetting> m = interpreterSettingManager.getInterpreterSettingTemplates();
return new JsonResponse<>(Status.OK, "", m).build();
}

49
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -31,12 +31,10 @@ import org.apache.shiro.web.env.EnvironmentLoaderListener;
import org.apache.shiro.web.servlet.ShiroFilter;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.helium.Helium;
import org.apache.zeppelin.helium.HeliumApplicationFactory;
import org.apache.zeppelin.helium.HeliumBundleFactory;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterOutput;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.notebook.Notebook;
@ -93,13 +91,11 @@ public class ZeppelinServer extends Application {
private NotebookRepoSync notebookRepo;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private DependencyResolver depResolver;
public ZeppelinServer() throws Exception {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
this.depResolver = new DependencyResolver(
conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
InterpreterOutput.limit = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT);
@ -129,27 +125,10 @@ public class ZeppelinServer extends Application {
new File(conf.getRelativeDir("zeppelin-web/src/app/spell")));
}
ZeppelinServer.helium = new Helium(
conf.getHeliumConfPath(),
conf.getHeliumRegistry(),
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
"helium-registry-cache"),
heliumBundleFactory,
heliumApplicationFactory);
// create bundle
try {
heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
this.schedulerFactory = new SchedulerFactory();
this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver,
new InterpreterOption(true));
this.replFactory = new InterpreterFactory(conf, notebookWsServer,
notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(),
interpreterSettingManager);
this.schedulerFactory = SchedulerFactory.singleton();
this.interpreterSettingManager = new InterpreterSettingManager(conf, notebookWsServer,
notebookWsServer, notebookWsServer);
this.replFactory = new InterpreterFactory(interpreterSettingManager);
this.notebookRepo = new NotebookRepoSync(conf);
this.noteSearchService = new LuceneSearch();
this.notebookAuthorization = NotebookAuthorization.init(conf);
@ -158,6 +137,22 @@ public class ZeppelinServer extends Application {
notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
noteSearchService, notebookAuthorization, credentials);
ZeppelinServer.helium = new Helium(
conf.getHeliumConfPath(),
conf.getHeliumRegistry(),
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
"helium-registry-cache"),
heliumBundleFactory,
heliumApplicationFactory,
interpreterSettingManager);
// create bundle
try {
heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
// to update notebook from application event from remote process.
heliumApplicationFactory.setNotebook(notebook);
// to update fire websocket event on application event.
@ -206,7 +201,7 @@ public class ZeppelinServer extends Application {
LOG.info("Shutting down Zeppelin Server ... ");
try {
jettyWebServer.stop();
notebook.getInterpreterSettingManager().shutdown();
notebook.getInterpreterSettingManager().close();
notebook.close();
Thread.sleep(3000);
} catch (Exception e) {

35
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -41,12 +41,7 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.helium.HeliumPackage;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@ -59,7 +54,6 @@ import org.apache.zeppelin.notebook.NotebookEventListener;
import org.apache.zeppelin.notebook.NotebookImportDeserializer;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.ParagraphJobListener;
import org.apache.zeppelin.notebook.ParagraphRuntimeInfo;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
@ -86,8 +80,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Queues;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken;
/**
@ -463,7 +455,8 @@ public class NotebookServer extends WebSocketServlet
Notebook notebook = notebook();
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
List<String> ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId());
List<String> ids = notebook.getInterpreterSettingManager()
.getInterpreterBinding(note.getId());
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(note.getId(), m);
@ -1022,7 +1015,7 @@ public class NotebookServer extends WebSocketServlet
List<String> interpreterSettingIds = new LinkedList<>();
interpreterSettingIds.add(defaultInterpreterId);
for (String interpreterSettingId : notebook.getInterpreterSettingManager().
getDefaultInterpreterSettingList()) {
getInterpreterSettingIds()) {
if (!interpreterSettingId.equals(defaultInterpreterId)) {
interpreterSettingIds.add(interpreterSettingId);
}
@ -1382,12 +1375,13 @@ public class NotebookServer extends WebSocketServlet
List<InterpreterSetting> settings =
notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(user, note.getId()) == null) {
if (setting.getOrCreateInterpreterGroup(user, note.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, note.getId())
.getId())) {
AngularObjectRegistry angularObjectRegistry =
setting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
setting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@ -1424,12 +1418,13 @@ public class NotebookServer extends WebSocketServlet
List<InterpreterSetting> settings =
notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(user, n.getId()) == null) {
if (setting.getOrCreateInterpreterGroup(user, n.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, n.getId())
.getId())) {
AngularObjectRegistry angularObjectRegistry =
setting.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
setting.getOrCreateInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
this.broadcastExcept(n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId())
@ -2302,13 +2297,13 @@ public class NotebookServer extends WebSocketServlet
for (InterpreterSetting intpSetting : settings) {
AngularObjectRegistry registry =
intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
conn.send(serializeMessage(
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
.put("interpreterGroupId",
intpSetting.getInterpreterGroup(user, note.getId()).getId())
intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getId())
.put("noteId", note.getId()).put("paragraphId", object.getParagraphId())));
}
}
@ -2354,7 +2349,7 @@ public class NotebookServer extends WebSocketServlet
}
List<String> settingIds =
notebook.getInterpreterSettingManager().getInterpreters(note.getId());
notebook.getInterpreterSettingManager().getInterpreterBinding(note.getId());
for (String id : settingIds) {
if (interpreterGroupId.contains(id)) {
broadcast(note.getId(),

7
zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
View file

@ -307,10 +307,9 @@ public abstract class AbstractTestRestApi {
protected static void shutDown() throws Exception {
if (!wasRunning) {
// restart interpreter to stop all interpreter processes
List<String> settingList = ZeppelinServer.notebook.getInterpreterSettingManager()
.getDefaultInterpreterSettingList();
for (String setting : settingList) {
ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting);
List<InterpreterSetting> settingList = ZeppelinServer.notebook.getInterpreterSettingManager().get();
for (InterpreterSetting setting : settingList) {
ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId());
}
if (shiroIni != null) {
FileUtils.deleteQuietly(shiroIni);

6
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -80,7 +80,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
// then
assertThat(get, isAllowed());
assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getAvailableInterpreterSettings().size(),
assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettingTemplates().size(),
body.entrySet().size());
get.releaseConnection();
}
@ -110,7 +110,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testSettingsCRUD() throws IOException {
// when: call create setting API
String rawRequest = "{\"name\":\"md2\",\"group\":\"md\"," +
String rawRequest = "{\"name\":\"md3\",\"group\":\"md\"," +
"\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
@ -367,7 +367,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testGetMetadataInfo() throws IOException {
String jsonRequest = "{\"name\":\"spark\",\"group\":\"spark\"," +
String jsonRequest = "{\"name\":\"spark_new\",\"group\":\"spark\"," +
"\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +

13
zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
View file

@ -30,6 +30,7 @@ import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.apache.zeppelin.rest.AbstractTestRestApi;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
@ -95,7 +96,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
}
@Test
public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException {
public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException, InterruptedException {
// create a notebook
Note note1 = notebook.createNote(anonymous);
@ -104,7 +105,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("md")) {
interpreterGroup = setting.getInterpreterGroup("anonymous", "sharedProcess");
interpreterGroup = setting.getOrCreateInterpreterGroup("anonymous", "sharedProcess");
break;
}
}
@ -115,6 +116,14 @@ public class NotebookServerTest extends AbstractTestRestApi {
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
// wait for paragraph finished
while(true) {
if (p1.getStatus() == Job.Status.FINISHED) {
break;
}
Thread.sleep(100);
}
// add angularObject
interpreterGroup.getAngularObjectRegistry().add("object1", "value1", note1.getId(), null);

1
zeppelin-server/src/test/resources/log4j.properties
View file

@ -33,7 +33,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
log4j.logger.org.apache.zeppelin.scheduler=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN

22
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -17,6 +17,14 @@
package org.apache.zeppelin.conf;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;
import org.apache.commons.configuration.tree.ConfigurationNode;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
import java.util.Arrays;
@ -24,15 +32,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;
import org.apache.commons.configuration.tree.ConfigurationNode;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.notebook.repo.GitNotebookRepo;
import org.apache.zeppelin.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Zeppelin configuration.
*
@ -528,7 +527,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
for (ZeppelinConfiguration.ConfVars v : ZeppelinConfiguration.ConfVars.values()) {
for (ConfVars v : ConfVars.values()) {
String key = v.getVarName();
if (!predicate.apply(key)) {
@ -653,7 +652,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_MONGO_COLLECTION("zeppelin.notebook.mongo.collection", "notes"),
ZEPPELIN_NOTEBOOK_MONGO_URI("zeppelin.notebook.mongo.uri", "mongodb://localhost"),
ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT("zeppelin.notebook.mongo.autoimport", false),
ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", GitNotebookRepo.class.getName()),
ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage",
"org.apache.zeppelin.notebook.repo.GitNotebookRepo"),
ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC("zeppelin.notebook.one.way.sync", false),
// whether by default note is public or private
ZEPPELIN_NOTEBOOK_PUBLIC("zeppelin.notebook.public", true),

54
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
View file

@ -16,14 +16,17 @@
*/
package org.apache.zeppelin.helium;
import com.google.gson.Gson;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.resource.DistributedResourcePool;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.resource.ResourceSet;
import org.apache.zeppelin.resource.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -47,19 +50,22 @@ public class Helium {
private final HeliumBundleFactory bundleFactory;
private final HeliumApplicationFactory applicationFactory;
private final InterpreterSettingManager interpreterSettingManager;
public Helium(
String heliumConfPath,
String registryPaths,
File registryCacheDir,
HeliumBundleFactory bundleFactory,
HeliumApplicationFactory applicationFactory)
HeliumApplicationFactory applicationFactory,
InterpreterSettingManager interpreterSettingManager)
throws IOException {
this.heliumConfPath = heliumConfPath;
this.registryPaths = registryPaths;
this.registryCacheDir = registryCacheDir;
this.bundleFactory = bundleFactory;
this.applicationFactory = applicationFactory;
this.interpreterSettingManager = interpreterSettingManager;
heliumConf = loadConf(heliumConfPath);
allPackages = getAllPackageInfo();
}
@ -350,7 +356,7 @@ public class Helium {
allResources = resourcePool.getAll();
}
} else {
allResources = ResourcePoolUtils.getAllResources();
allResources = interpreterSettingManager.getAllResources();
}
for (List<HeliumPackageSearchResult> pkgs : allPackages.values()) {
@ -478,4 +484,40 @@ public class Helium {
return mixed;
}
public ResourceSet getAllResources() {
return getAllResourcesExcept(null);
}
private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
ResourceSet resourceSet = new ResourceSet();
for (ManagedInterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
if (interpreterGroupExcludsion != null &&
intpGroup.getId().equals(interpreterGroupExcludsion)) {
continue;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
} else if (remoteInterpreterProcess.isRunning()) {
List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
@Override
public List<String> call(RemoteInterpreterService.Client client) throws Exception {
return client.resourcePoolGetAll();
}
}
);
Gson gson = new Gson();
for (String res : resourceList) {
resourceSet.add(gson.fromJson(res, Resource.class));
}
}
}
return resourceSet;
}
}

125
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
View file

@ -16,7 +16,6 @@
*/
package org.apache.zeppelin.helium;
import org.apache.thrift.TException;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
@ -80,7 +79,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
try {
// get interpreter process
Interpreter intp = paragraph.getRepl(paragraph.getRequiredReplName());
InterpreterGroup intpGroup = intp.getInterpreterGroup();
ManagedInterpreterGroup intpGroup = (ManagedInterpreterGroup) intp.getInterpreterGroup();
RemoteInterpreterProcess intpProcess = intpGroup.getRemoteInterpreterProcess();
if (intpProcess == null) {
throw new ApplicationException("Target interpreter process is not running");
@ -105,38 +104,33 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
private void load(RemoteInterpreterProcess intpProcess, ApplicationState appState)
throws Exception {
RemoteInterpreterService.Client client = null;
synchronized (appState) {
if (appState.getStatus() == ApplicationState.Status.LOADED) {
// already loaded
return;
}
try {
appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING);
String pkgInfo = pkg.toJson();
String appId = appState.getId();
appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING);
final String pkgInfo = pkg.toJson();
final String appId = appState.getId();
client = intpProcess.getClient();
RemoteApplicationResult ret = client.loadApplication(
appId,
pkgInfo,
paragraph.getNote().getId(),
paragraph.getId());
if (ret.isSuccess()) {
appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED);
} else {
throw new ApplicationException(ret.getMsg());
}
} catch (TException e) {
intpProcess.releaseBrokenClient(client);
throw e;
} finally {
if (client != null) {
intpProcess.releaseClient(client);
}
RemoteApplicationResult ret = intpProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
@Override
public RemoteApplicationResult call(RemoteInterpreterService.Client client)
throws Exception {
return client.loadApplication(
appId,
pkgInfo,
paragraph.getNote().getId(),
paragraph.getId());
}
}
);
if (ret.isSuccess()) {
appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED);
} else {
throw new ApplicationException(ret.getMsg());
}
}
}
@ -199,7 +193,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
}
private void unload(ApplicationState appsToUnload) throws ApplicationException {
private void unload(final ApplicationState appsToUnload) throws ApplicationException {
synchronized (appsToUnload) {
if (appsToUnload.getStatus() != ApplicationState.Status.LOADED) {
throw new ApplicationException(
@ -212,31 +206,24 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
RemoteInterpreterProcess intpProcess =
intp.getInterpreterGroup().getRemoteInterpreterProcess();
((ManagedInterpreterGroup) intp.getInterpreterGroup()).getRemoteInterpreterProcess();
if (intpProcess == null) {
throw new ApplicationException("Target interpreter process is not running");
}
RemoteInterpreterService.Client client;
try {
client = intpProcess.getClient();
} catch (Exception e) {
throw new ApplicationException(e);
}
try {
RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId());
if (ret.isSuccess()) {
appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED);
} else {
throw new ApplicationException(ret.getMsg());
}
} catch (TException e) {
intpProcess.releaseBrokenClient(client);
throw new ApplicationException(e);
} finally {
intpProcess.releaseClient(client);
RemoteApplicationResult ret = intpProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
@Override
public RemoteApplicationResult call(RemoteInterpreterService.Client client)
throws Exception {
return client.unloadApplication(appsToUnload.getId());
}
}
);
if (ret.isSuccess()) {
appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED);
} else {
throw new ApplicationException(ret.getMsg());
}
}
}
@ -286,7 +273,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
}
private void run(ApplicationState app) throws ApplicationException {
private void run(final ApplicationState app) throws ApplicationException {
synchronized (app) {
if (app.getStatus() != ApplicationState.Status.LOADED) {
throw new ApplicationException(
@ -299,33 +286,23 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
RemoteInterpreterProcess intpProcess =
intp.getInterpreterGroup().getRemoteInterpreterProcess();
((ManagedInterpreterGroup) intp.getInterpreterGroup()).getRemoteInterpreterProcess();
if (intpProcess == null) {
throw new ApplicationException("Target interpreter process is not running");
}
RemoteInterpreterService.Client client = null;
try {
client = intpProcess.getClient();
} catch (Exception e) {
throw new ApplicationException(e);
}
try {
RemoteApplicationResult ret = client.runApplication(app.getId());
if (ret.isSuccess()) {
// success
} else {
throw new ApplicationException(ret.getMsg());
}
} catch (TException e) {
intpProcess.releaseBrokenClient(client);
client = null;
throw new ApplicationException(e);
} finally {
if (client != null) {
intpProcess.releaseClient(client);
}
RemoteApplicationResult ret = intpProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
@Override
public RemoteApplicationResult call(RemoteInterpreterService.Client client)
throws Exception {
return client.runApplication(app.getId());
}
}
);
if (ret.isSuccess()) {
// success
} else {
throw new ApplicationException(ret.getMsg());
}
}
}

352
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
View file

@ -17,288 +17,31 @@
package org.apache.zeppelin.interpreter;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.NullArgumentException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* //TODO(zjffdu) considering to move to InterpreterSettingManager
*
* Manage interpreters.
*/
public class InterpreterFactory implements InterpreterGroupFactory {
private static final Logger logger = LoggerFactory.getLogger(InterpreterFactory.class);
private Map<String, URLClassLoader> cleanCl =
Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
private ZeppelinConfiguration conf;
public class InterpreterFactory {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterFactory.class);
private final InterpreterSettingManager interpreterSettingManager;
private AngularObjectRegistryListener angularObjectRegistryListener;
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private final ApplicationEventListener appEventListener;
private boolean shiroEnabled;
private Map<String, String> env = new HashMap<>();
private Interpreter devInterpreter;
public InterpreterFactory(ZeppelinConfiguration conf,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appEventListener, DependencyResolver depResolver,
boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager)
throws InterpreterException, IOException, RepositoryException {
this.conf = conf;
this.angularObjectRegistryListener = angularObjectRegistryListener;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.appEventListener = appEventListener;
this.shiroEnabled = shiroEnabled;
public InterpreterFactory(InterpreterSettingManager interpreterSettingManager) {
this.interpreterSettingManager = interpreterSettingManager;
//TODO(jl): Fix it not to use InterpreterGroupFactory
interpreterSettingManager.setInterpreterGroupFactory(this);
logger.info("shiroEnabled: {}", shiroEnabled);
}
/**
* @param id interpreterGroup id. Combination of interpreterSettingId + noteId/userId/shared
* depends on interpreter mode
*/
@Override
public InterpreterGroup createInterpreterGroup(String id, InterpreterOption option)
throws InterpreterException, NullArgumentException {
//When called from REST API without option we receive NPE
if (option == null) {
throw new NullArgumentException("option");
}
AngularObjectRegistry angularObjectRegistry;
InterpreterGroup interpreterGroup = new InterpreterGroup(id);
if (option.isRemote()) {
angularObjectRegistry =
new RemoteAngularObjectRegistry(id, angularObjectRegistryListener, interpreterGroup);
} else {
angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener);
// TODO(moon) : create distributed resource pool for local interpreters and set
}
interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
return interpreterGroup;
}
public void createInterpretersForNote(InterpreterSetting interpreterSetting, String user,
String noteId, String interpreterSessionKey) {
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
InterpreterOption option = interpreterSetting.getOption();
Properties properties = interpreterSetting.getFlatProperties();
// if interpreters are already there, wait until they're being removed
synchronized (interpreterGroup) {
long interpreterRemovalWaitStart = System.nanoTime();
// interpreter process supposed to be terminated by RemoteInterpreterProcess.dereference()
// in ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT msec. However, if termination of the process and
// removal from interpreter group take too long, throw an error.
long minTimeout = 10L * 1000 * 1000000; // 10 sec
long interpreterRemovalWaitTimeout = Math.max(minTimeout,
conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT) * 1000000L * 2);
while (interpreterGroup.containsKey(interpreterSessionKey)) {
if (System.nanoTime() - interpreterRemovalWaitStart > interpreterRemovalWaitTimeout) {
throw new InterpreterException("Can not create interpreter");
}
try {
interpreterGroup.wait(1000);
} catch (InterruptedException e) {
logger.debug(e.getMessage(), e);
}
}
}
logger.info("Create interpreter instance {} for note {}", interpreterSetting.getName(), noteId);
List<InterpreterInfo> interpreterInfos = interpreterSetting.getInterpreterInfos();
String path = interpreterSetting.getPath();
InterpreterRunner runner = interpreterSetting.getInterpreterRunner();
Interpreter interpreter;
for (InterpreterInfo info : interpreterInfos) {
if (option.isRemote()) {
if (option.isExistingProcess()) {
interpreter =
connectToRemoteRepl(interpreterSessionKey, info.getClassName(), option.getHost(),
option.getPort(), properties, interpreterSetting.getId(), user,
option.isUserImpersonate);
} else {
interpreter = createRemoteRepl(path, interpreterSessionKey, info.getClassName(),
properties, interpreterSetting.getId(), user, option.isUserImpersonate(), runner);
}
} else {
interpreter = createRepl(interpreterSetting.getPath(), info.getClassName(), properties);
}
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(interpreterSessionKey);
if (null == interpreters) {
interpreters = new ArrayList<>();
interpreterGroup.put(interpreterSessionKey, interpreters);
}
if (info.isDefaultInterpreter()) {
interpreters.add(0, interpreter);
} else {
interpreters.add(interpreter);
}
}
logger.info("Interpreter {} {} created", interpreter.getClassName(), interpreter.hashCode());
interpreter.setInterpreterGroup(interpreterGroup);
}
}
private Interpreter createRepl(String dirName, String className, Properties property)
throws InterpreterException {
logger.info("Create repl {} from {}", className, dirName);
ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
try {
URLClassLoader ccl = cleanCl.get(dirName);
if (ccl == null) {
// classloader fallback
ccl = URLClassLoader.newInstance(new URL[]{}, oldcl);
}
boolean separateCL = true;
try { // check if server's classloader has driver already.
Class cls = this.getClass().forName(className);
if (cls != null) {
separateCL = false;
}
} catch (Exception e) {
logger.error("exception checking server classloader driver", e);
}
URLClassLoader cl;
if (separateCL == true) {
cl = URLClassLoader.newInstance(new URL[]{}, ccl);
} else {
cl = ccl;
}
Thread.currentThread().setContextClassLoader(cl);
Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className);
Constructor<Interpreter> constructor =
replClass.getConstructor(new Class[]{Properties.class});
Interpreter repl = constructor.newInstance(property);
repl.setClassloaderUrls(ccl.getURLs());
LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl));
return intp;
} catch (SecurityException e) {
throw new InterpreterException(e);
} catch (NoSuchMethodException e) {
throw new InterpreterException(e);
} catch (IllegalArgumentException e) {
throw new InterpreterException(e);
} catch (InstantiationException e) {
throw new InterpreterException(e);
} catch (IllegalAccessException e) {
throw new InterpreterException(e);
} catch (InvocationTargetException e) {
throw new InterpreterException(e);
} catch (ClassNotFoundException e) {
throw new InterpreterException(e);
} finally {
Thread.currentThread().setContextClassLoader(oldcl);
}
}
private Interpreter connectToRemoteRepl(String interpreterSessionKey, String className,
String host, int port, Properties property, String interpreterSettingId, String userName,
Boolean isUserImpersonate) {
int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
LazyOpenInterpreter intp = new LazyOpenInterpreter(
new RemoteInterpreter(property, interpreterSessionKey, className, host, port, localRepoPath,
connectTimeout, maxPoolSize, remoteInterpreterProcessListener, appEventListener,
userName, isUserImpersonate, conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT)));
return intp;
}
Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKey,
String className, Properties property, String interpreterSettingId,
String userName, Boolean isUserImpersonate, InterpreterRunner interpreterRunner) {
int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
String interpreterRunnerPath;
String interpreterGroupName = interpreterSettingManager.get(interpreterSettingId).getName();
if (null != interpreterRunner) {
interpreterRunnerPath = interpreterRunner.getPath();
Path p = Paths.get(interpreterRunnerPath);
if (!p.isAbsolute()) {
interpreterRunnerPath = Joiner.on(File.separator)
.join(interpreterPath, interpreterRunnerPath);
}
} else {
interpreterRunnerPath = conf.getInterpreterRemoteRunnerPath();
}
RemoteInterpreter remoteInterpreter =
new RemoteInterpreter(property, interpreterSessionKey, className,
interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize,
remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate,
conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName);
remoteInterpreter.addEnv(env);
return new LazyOpenInterpreter(remoteInterpreter);
}
private List<Interpreter> createOrGetInterpreterList(String user, String noteId,
InterpreterSetting setting) {
InterpreterGroup interpreterGroup = setting.getInterpreterGroup(user, noteId);
synchronized (interpreterGroup) {
String interpreterSessionKey =
interpreterSettingManager.getInterpreterSessionKey(user, noteId, setting);
if (!interpreterGroup.containsKey(interpreterSessionKey)) {
createInterpretersForNote(setting, user, noteId, interpreterSessionKey);
}
return interpreterGroup.get(interpreterSessionKey);
}
}
private InterpreterSetting getInterpreterSettingByGroup(List<InterpreterSetting> settings,
String group) {
Preconditions.checkNotNull(group, "group should be not null");
Preconditions.checkNotNull(group, "group should be not null");
for (InterpreterSetting setting : settings) {
if (group.equals(setting.getName())) {
return setting;
@ -307,80 +50,41 @@ public class InterpreterFactory implements InterpreterGroupFactory {
return null;
}
private String getInterpreterClassFromInterpreterSetting(InterpreterSetting setting,
String name) {
Preconditions.checkNotNull(name, "name should be not null");
for (InterpreterInfo info : setting.getInterpreterInfos()) {
String infoName = info.getName();
if (null != info.getName() && name.equals(infoName)) {
return info.getClassName();
}
}
return null;
}
private Interpreter getInterpreter(String user, String noteId, InterpreterSetting setting,
String name) {
Preconditions.checkNotNull(noteId, "noteId should be not null");
Preconditions.checkNotNull(setting, "setting should be not null");
Preconditions.checkNotNull(name, "name should be not null");
String className;
if (null != (className = getInterpreterClassFromInterpreterSetting(setting, name))) {
List<Interpreter> interpreterGroup = createOrGetInterpreterList(user, noteId, setting);
for (Interpreter interpreter : interpreterGroup) {
if (className.equals(interpreter.getClassName())) {
return interpreter;
}
}
}
return null;
}
public Interpreter getInterpreter(String user, String noteId, String replName) {
List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(noteId);
InterpreterSetting setting;
Interpreter interpreter;
if (settings == null || settings.size() == 0) {
LOGGER.error("No interpreter is binded to this note: " + noteId);
return null;
}
if (replName == null || replName.trim().length() == 0) {
// get default settings (first available)
// TODO(jl): Fix it in case of returning null
InterpreterSetting defaultSettings = interpreterSettingManager
.getDefaultInterpreterSetting(settings);
return createOrGetInterpreterList(user, noteId, defaultSettings).get(0);
if (StringUtils.isBlank(replName)) {
// Get the default interpreter of the first interpreter binding
InterpreterSetting defaultSetting = settings.get(0);
return defaultSetting.getDefaultInterpreter(user, noteId);
}
String[] replNameSplit = replName.split("\\.");
if (replNameSplit.length == 2) {
String group = null;
String name = null;
group = replNameSplit[0];
name = replNameSplit[1];
String group = replNameSplit[0];
String name = replNameSplit[1];
setting = getInterpreterSettingByGroup(settings, group);
if (null != setting) {
interpreter = getInterpreter(user, noteId, setting, name);
interpreter = setting.getInterpreter(user, noteId, name);
if (null != interpreter) {
return interpreter;
}
}
throw new InterpreterException(replName + " interpreter not found");
} else {
// first assume replName is 'name' of interpreter. ('groupName' is ommitted)
// search 'name' from first (default) interpreter group
// TODO(jl): Handle with noteId to support defaultInterpreter per note.
setting = interpreterSettingManager.getDefaultInterpreterSetting(settings);
interpreter = getInterpreter(user, noteId, setting, replName);
setting = settings.get(0);
interpreter = setting.getInterpreter(user, noteId, replName);
if (null != interpreter) {
return interpreter;
@ -391,33 +95,17 @@ public class InterpreterFactory implements InterpreterGroupFactory {
setting = getInterpreterSettingByGroup(settings, replName);
if (null != setting) {
List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, setting);
if (null != interpreters) {
return interpreters.get(0);
}
return setting.getDefaultInterpreter(user, noteId);
}
// Support the legacy way to use it
for (InterpreterSetting s : settings) {
if (s.getGroup().equals(replName)) {
List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, s);
if (null != interpreters) {
return interpreters.get(0);
}
return setting.getDefaultInterpreter(user, noteId);
}
}
}
//TODO(zjffdu) throw InterpreterException instead of return null
return null;
}
public Map<String, String> getEnv() {
return env;
}
public void setEnv(Map<String, String> env) {
this.env = env;
}
}

68
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
View file

@ -19,22 +19,78 @@ package org.apache.zeppelin.interpreter;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.internal.StringMap;
import org.apache.commons.io.IOUtils;
import org.apache.zeppelin.common.JsonSerializable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.repository.RemoteRepository;
import java.util.List;
import java.util.Map;
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.PosixFilePermission;
import java.util.*;
import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
/**
*
*/
public class InterpreterInfoSaving implements JsonSerializable {
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterInfoSaving.class);
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
public Map<String, InterpreterSetting> interpreterSettings;
public Map<String, List<String>> interpreterBindings;
public List<RemoteRepository> interpreterRepositories;
public Map<String, InterpreterSetting> interpreterSettings = new HashMap<>();
public Map<String, List<String>> interpreterBindings = new HashMap<>();
public List<RemoteRepository> interpreterRepositories = new ArrayList<>();
public static InterpreterInfoSaving loadFromFile(Path file) throws IOException {
LOGGER.info("Load interpreter setting from file: " + file);
InterpreterInfoSaving infoSaving = null;
try (BufferedReader json = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
JsonParser jsonParser = new JsonParser();
JsonObject jsonObject = jsonParser.parse(json).getAsJsonObject();
infoSaving = InterpreterInfoSaving.fromJson(jsonObject.toString());
if (infoSaving != null && infoSaving.interpreterSettings != null) {
for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) {
// Always use separate interpreter process
// While we decided to turn this feature on always (without providing
// enable/disable option on GUI).
// previously created setting should turn this feature on here.
interpreterSetting.getOption().setRemote(true);
interpreterSetting.convertPermissionsFromUsersToOwners(
jsonObject.getAsJsonObject("interpreterSettings")
.getAsJsonObject(interpreterSetting.getId()));
}
}
}
return infoSaving == null ? new InterpreterInfoSaving() : infoSaving;
}
public void saveToFile(Path file) throws IOException {
if (!Files.exists(file)) {
Files.createFile(file);
try {
Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
Files.setPosixFilePermissions(file, permissions);
} catch (UnsupportedOperationException e) {
// File system does not support Posix file permissions (likely windows) - continue anyway.
LOGGER.warn("unable to setPosixFilePermissions on '{}'.", file);
};
}
LOGGER.info("Save Interpreter Settings to " + file);
IOUtils.write(this.toJson(), new FileOutputStream(file.toFile()));
}
public String toJson() {
return gson.toJson(this);

831
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
View file

@ -17,8 +17,39 @@
package org.apache.zeppelin.interpreter;
import java.util.Arrays;
import java.util.Collection;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.annotations.SerializedName;
import com.google.gson.internal.StringMap;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@ -26,104 +57,253 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.zeppelin.dep.Dependency;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.annotations.SerializedName;
import com.google.gson.internal.StringMap;
import static org.apache.zeppelin.notebook.utility.IdHashes.generateId;
import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE;
import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
import static org.apache.zeppelin.util.IdHashes.generateId;
/**
* Interpreter settings
* Represent one InterpreterSetting in the interpreter setting page
*/
public class InterpreterSetting {
private static final Logger logger = LoggerFactory.getLogger(InterpreterSetting.class);
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSetting.class);
private static final String SHARED_PROCESS = "shared_process";
private static final String SHARED_SESSION = "shared_session";
private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
"language", (Object) "text",
"editOnDblClick", false);
private String id;
private String name;
// always be null in case of InterpreterSettingRef
// the original interpreter setting template name where it is created from
private String group;
//TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json
/**
* properties can be either Properties or Map<String, InterpreterProperty>
* properties should be:
* - Properties when Interpreter instances are saved to `conf/interpreter.json` file
* - Map<String, InterpreterProperty> when Interpreters are registered
* : this is needed after https://github.com/apache/zeppelin/pull/1145
* which changed the way of getting default interpreter setting AKA interpreterSettingsRef
* Note(mina): In order to simplify the implementation, I chose to change properties
* from Properties to Object instead of creating new classes.
*/
private Object properties = new Properties();
private Status status;
private String errorReason;
@SerializedName("interpreterGroup")
private List<InterpreterInfo> interpreterInfos;
private List<Dependency> dependencies = new ArrayList<>();
private InterpreterOption option = new InterpreterOption(true);
@SerializedName("runner")
private InterpreterRunner interpreterRunner;
///////////////////////////////////////////////////////////////////////////////////////////
private transient InterpreterSettingManager interpreterSettingManager;
private transient String interpreterDir;
private final transient Map<String, ManagedInterpreterGroup> interpreterGroups =
new ConcurrentHashMap<>();
private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock;
private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock;
private transient AngularObjectRegistryListener angularObjectRegistryListener;
private transient RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private transient ApplicationEventListener appEventListener;
private transient DependencyResolver dependencyResolver;
private transient Map<String, String> infos;
// Map of the note and paragraphs which has runtime infos generated by this interpreter setting.
// This map is used to clear the infos in paragraph when the interpretersetting is restarted
private transient Map<String, Set<String>> runtimeInfosToBeCleared;
private transient ZeppelinConfiguration conf = new ZeppelinConfiguration();
private transient Map<String, URLClassLoader> cleanCl =
Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
///////////////////////////////////////////////////////////////////////////////////////////
/**
* properties can be either Map<String, DefaultInterpreterProperty> or
* Map<String, InterpreterProperty>
* properties should be:
* - Map<String, InterpreterProperty> when Interpreter instances are saved to
* `conf/interpreter.json` file
* - Map<String, DefaultInterpreterProperty> when Interpreters are registered
* : this is needed after https://github.com/apache/zeppelin/pull/1145
* which changed the way of getting default interpreter setting AKA interpreterSettingsRef
* Builder class for InterpreterSetting
*/
private Object properties;
private Status status;
private String errorReason;
public static class Builder {
private InterpreterSetting interpreterSetting;
@SerializedName("interpreterGroup")
private List<InterpreterInfo> interpreterInfos;
private final transient Map<String, InterpreterGroup> interpreterGroupRef = new HashMap<>();
private List<Dependency> dependencies = new LinkedList<>();
private InterpreterOption option;
private transient String path;
public Builder() {
this.interpreterSetting = new InterpreterSetting();
}
@SerializedName("runner")
private InterpreterRunner interpreterRunner;
public Builder setId(String id) {
interpreterSetting.id = id;
return this;
}
@Deprecated
private transient InterpreterGroupFactory interpreterGroupFactory;
public Builder setName(String name) {
interpreterSetting.name = name;
return this;
}
private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock;
private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock;
public Builder setGroup(String group) {
interpreterSetting.group = group;
return this;
}
public Builder setInterpreterInfos(List<InterpreterInfo> interpreterInfos) {
interpreterSetting.interpreterInfos = interpreterInfos;
return this;
}
public Builder setProperties(Object properties) {
interpreterSetting.properties = properties;
return this;
}
public Builder setOption(InterpreterOption option) {
interpreterSetting.option = option;
return this;
}
public Builder setInterpreterDir(String interpreterDir) {
interpreterSetting.interpreterDir = interpreterDir;
return this;
}
public Builder setRunner(InterpreterRunner runner) {
interpreterSetting.interpreterRunner = runner;
return this;
}
public Builder setDependencies(List<Dependency> dependencies) {
interpreterSetting.dependencies = dependencies;
return this;
}
public Builder setConf(ZeppelinConfiguration conf) {
interpreterSetting.conf = conf;
return this;
}
public Builder setDependencyResolver(DependencyResolver dependencyResolver) {
interpreterSetting.dependencyResolver = dependencyResolver;
return this;
}
// public Builder setInterpreterRunner(InterpreterRunner runner) {
// interpreterSetting.interpreterRunner = runner;
// return this;
// }
public Builder setIntepreterSettingManager(
InterpreterSettingManager interpreterSettingManager) {
interpreterSetting.interpreterSettingManager = interpreterSettingManager;
return this;
}
public Builder setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener
remoteInterpreterProcessListener) {
interpreterSetting.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
return this;
}
public Builder setAngularObjectRegistryListener(
AngularObjectRegistryListener angularObjectRegistryListener) {
interpreterSetting.angularObjectRegistryListener = angularObjectRegistryListener;
return this;
}
public Builder setApplicationEventListener(ApplicationEventListener applicationEventListener) {
interpreterSetting.appEventListener = applicationEventListener;
return this;
}
public InterpreterSetting create() {
// post processing
interpreterSetting.postProcessing();
return interpreterSetting;
}
}
public InterpreterSetting() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.id = generateId();
interpreterGroupReadLock = lock.readLock();
interpreterGroupWriteLock = lock.writeLock();
}
public InterpreterSetting(String id, String name, String group,
List<InterpreterInfo> interpreterInfos, Object properties, List<Dependency> dependencies,
InterpreterOption option, String path, InterpreterRunner runner) {
this();
this.id = id;
this.name = name;
this.group = group;
this.interpreterInfos = interpreterInfos;
this.properties = properties;
this.dependencies = dependencies;
this.option = option;
this.path = path;
void postProcessing() {
this.status = Status.READY;
this.interpreterRunner = runner;
}
public InterpreterSetting(String name, String group, List<InterpreterInfo> interpreterInfos,
Object properties, List<Dependency> dependencies, InterpreterOption option, String path,
InterpreterRunner runner) {
this(generateId(), name, group, interpreterInfos, properties, dependencies, option, path,
runner);
}
/**
* Create interpreter from interpreterSettingRef
* Create interpreter from InterpreterSettingTemplate
*
* @param o interpreterSetting from interpreterSettingRef
* @param o interpreterSetting from InterpreterSettingTemplate
*/
public InterpreterSetting(InterpreterSetting o) {
this(generateId(), o.getName(), o.getGroup(), o.getInterpreterInfos(), o.getProperties(),
o.getDependencies(), o.getOption(), o.getPath(), o.getInterpreterRunner());
this();
this.id = generateId();
this.name = o.name;
this.group = o.group;
this.properties = convertInterpreterProperties(
(Map<String, DefaultInterpreterProperty>) o.getProperties());
this.interpreterInfos = new ArrayList<>(o.getInterpreterInfos());
this.option = InterpreterOption.fromInterpreterOption(o.getOption());
this.dependencies = new ArrayList<>(o.getDependencies());
this.interpreterDir = o.getInterpreterDir();
this.interpreterRunner = o.getInterpreterRunner();
this.conf = o.getConf();
}
public AngularObjectRegistryListener getAngularObjectRegistryListener() {
return angularObjectRegistryListener;
}
public RemoteInterpreterProcessListener getRemoteInterpreterProcessListener() {
return remoteInterpreterProcessListener;
}
public ApplicationEventListener getAppEventListener() {
return appEventListener;
}
public DependencyResolver getDependencyResolver() {
return dependencyResolver;
}
public InterpreterSettingManager getInterpreterSettingManager() {
return interpreterSettingManager;
}
public void setAngularObjectRegistryListener(AngularObjectRegistryListener
angularObjectRegistryListener) {
this.angularObjectRegistryListener = angularObjectRegistryListener;
}
public void setAppEventListener(ApplicationEventListener appEventListener) {
this.appEventListener = appEventListener;
}
public void setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener
remoteInterpreterProcessListener) {
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
}
public void setDependencyResolver(DependencyResolver dependencyResolver) {
this.dependencyResolver = dependencyResolver;
}
public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
this.interpreterSettingManager = interpreterSettingManager;
}
public String getId() {
@ -138,10 +318,9 @@ public class InterpreterSetting {
return group;
}
private String getInterpreterProcessKey(String user, String noteId) {
InterpreterOption option = getOption();
private String getInterpreterGroupId(String user, String noteId) {
String key;
if (getOption().isExistingProcess) {
if (option.isExistingProcess) {
key = Constants.EXISTING_PROCESS;
} else if (getOption().isProcess()) {
key = (option.perUserIsolated() ? user : "") + ":" + (option.perNoteIsolated() ? noteId : "");
@ -149,40 +328,11 @@ public class InterpreterSetting {
key = SHARED_PROCESS;
}
//logger.debug("getInterpreterProcessKey: {} for InterpreterSetting Id: {}, Name: {}",
// key, getId(), getName());
return key;
//TODO(zjffdu) we encode interpreter setting id into groupId, this is not a good design
return id + ":" + key;
}
private boolean isEqualInterpreterKeyProcessKey(String refKey, String processKey) {
InterpreterOption option = getOption();
int validCount = 0;
if (getOption().isProcess()
&& !(option.perUserIsolated() == true && option.perNoteIsolated() == true)) {
List<String> processList = Arrays.asList(processKey.split(":"));
List<String> refList = Arrays.asList(refKey.split(":"));
if (refList.size() <= 1 || processList.size() <= 1) {
return refKey.equals(processKey);
}
if (processList.get(0).equals("") || processList.get(0).equals(refList.get(0))) {
validCount = validCount + 1;
}
if (processList.get(1).equals("") || processList.get(1).equals(refList.get(1))) {
validCount = validCount + 1;
}
return (validCount >= 2);
} else {
return refKey.equals(processKey);
}
}
String getInterpreterSessionKey(String user, String noteId) {
InterpreterOption option = getOption();
private String getInterpreterSessionId(String user, String noteId) {
String key;
if (option.isExistingProcess()) {
key = Constants.EXISTING_PROCESS;
@ -193,120 +343,153 @@ public class InterpreterSetting {
} else if (option.perNoteScoped()) {
key = noteId;
} else {
key = "shared_session";
key = SHARED_SESSION;
}
logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
"{}", key, noteId, user, getName());
return key;
}
public InterpreterGroup getInterpreterGroup(String user, String noteId) {
String key = getInterpreterProcessKey(user, noteId);
if (!interpreterGroupRef.containsKey(key)) {
String interpreterGroupId = getId() + ":" + key;
InterpreterGroup intpGroup =
interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption());
public ManagedInterpreterGroup getOrCreateInterpreterGroup(String user, String noteId) {
String groupId = getInterpreterGroupId(user, noteId);
try {
interpreterGroupWriteLock.lock();
logger.debug("create interpreter group with groupId:" + interpreterGroupId);
interpreterGroupRef.put(key, intpGroup);
interpreterGroupWriteLock.unlock();
if (!interpreterGroups.containsKey(groupId)) {
LOGGER.info("Create InterpreterGroup with groupId {} for user {} and note {}",
groupId, user, noteId);
ManagedInterpreterGroup intpGroup = createInterpreterGroup(groupId);
interpreterGroups.put(groupId, intpGroup);
}
return interpreterGroups.get(groupId);
} finally {
interpreterGroupWriteLock.unlock();;
}
}
void removeInterpreterGroup(String groupId) {
this.interpreterGroups.remove(groupId);
}
ManagedInterpreterGroup getInterpreterGroup(String user, String noteId) {
String groupId = getInterpreterGroupId(user, noteId);
try {
interpreterGroupReadLock.lock();
return interpreterGroupRef.get(key);
return interpreterGroups.get(groupId);
} finally {
interpreterGroupReadLock.unlock();;
}
}
ManagedInterpreterGroup getInterpreterGroup(String groupId) {
return interpreterGroups.get(groupId);
}
@VisibleForTesting
public ArrayList<ManagedInterpreterGroup> getAllInterpreterGroups() {
try {
interpreterGroupReadLock.lock();
return new ArrayList(interpreterGroups.values());
} finally {
interpreterGroupReadLock.unlock();
}
}
public Collection<InterpreterGroup> getAllInterpreterGroups() {
try {
interpreterGroupReadLock.lock();
return new LinkedList<>(interpreterGroupRef.values());
} finally {
interpreterGroupReadLock.unlock();
}
}
void closeAndRemoveInterpreterGroup(String noteId, String user) {
if (user.equals("anonymous")) {
user = "";
}
String processKey = getInterpreterProcessKey(user, noteId);
String sessionKey = getInterpreterSessionKey(user, noteId);
List<InterpreterGroup> groupToRemove = new LinkedList<>();
InterpreterGroup groupItem;
for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
if (isEqualInterpreterKeyProcessKey(intpKey, processKey)) {
interpreterGroupWriteLock.lock();
// TODO(jl): interpreterGroup has two or more sessionKeys inside it. thus we should not
// remove interpreterGroup if it has two or more values.
groupItem = interpreterGroupRef.get(intpKey);
interpreterGroupWriteLock.unlock();
groupToRemove.add(groupItem);
}
for (InterpreterGroup groupToClose : groupToRemove) {
// TODO(jl): Fix the logic removing session. Now, it's handled into groupToClose.clsose()
groupToClose.close(interpreterGroupRef, intpKey, sessionKey);
}
groupToRemove.clear();
}
//Remove session because all interpreters in this session are closed
//TODO(jl): Change all code to handle interpreter one by one or all at once
}
void closeAndRemoveAllInterpreterGroups() {
for (String processKey : new HashSet<>(interpreterGroupRef.keySet())) {
InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
for (String sessionKey : new HashSet<>(interpreterGroup.keySet())) {
interpreterGroup.close(interpreterGroupRef, processKey, sessionKey);
Map<String, Object> getEditorFromSettingByClassName(String className) {
for (InterpreterInfo intpInfo : interpreterInfos) {
if (className.equals(intpInfo.getClassName())) {
if (intpInfo.getEditor() == null) {
break;
}
return intpInfo.getEditor();
}
}
return DEFAULT_EDITOR;
}
void shutdownAndRemoveAllInterpreterGroups() {
for (InterpreterGroup interpreterGroup : interpreterGroupRef.values()) {
interpreterGroup.shutdown();
void closeInterpreters(String user, String noteId) {
ManagedInterpreterGroup interpreterGroup = getInterpreterGroup(user, noteId);
if (interpreterGroup != null) {
String sessionId = getInterpreterSessionId(user, noteId);
interpreterGroup.close(sessionId);
}
}
public void close() {
LOGGER.info("Close InterpreterSetting: " + name);
for (ManagedInterpreterGroup intpGroup : interpreterGroups.values()) {
intpGroup.close();
}
interpreterGroups.clear();
this.runtimeInfosToBeCleared = null;
this.infos = null;
}
public void setProperties(Object object) {
if (object instanceof StringMap) {
StringMap<String> map = (StringMap) properties;
Properties newProperties = new Properties();
for (String key : map.keySet()) {
newProperties.put(key, map.get(key));
}
this.properties = newProperties;
} else {
this.properties = object;
}
}
public Object getProperties() {
return properties;
}
public Properties getFlatProperties() {
Properties p = new Properties();
if (properties != null) {
Map<String, InterpreterProperty> propertyMap = (Map<String, InterpreterProperty>) properties;
for (String key : propertyMap.keySet()) {
InterpreterProperty tmp = propertyMap.get(key);
p.put(tmp.getName() != null ? tmp.getName() : key,
tmp.getValue() != null ? tmp.getValue().toString() : null);
}
@VisibleForTesting
public void setProperty(String name, String value) {
((Map<String, InterpreterProperty>) properties).put(name, new InterpreterProperty(name, value));
}
// This method is supposed to be only called by InterpreterSetting
// but not InterpreterSetting Template
public Properties getJavaProperties() {
Properties jProperties = new Properties();
Map<String, InterpreterProperty> iProperties = (Map<String, InterpreterProperty>) properties;
for (Map.Entry<String, InterpreterProperty> entry : iProperties.entrySet()) {
jProperties.setProperty(entry.getKey(), entry.getValue().getValue().toString());
}
return p;
if (!jProperties.containsKey("zeppelin.interpreter.output.limit")) {
jProperties.setProperty("zeppelin.interpreter.output.limit",
conf.getInt(ZEPPELIN_INTERPRETER_OUTPUT_LIMIT) + "");
}
if (!jProperties.containsKey("zeppelin.interpreter.max.poolsize")) {
jProperties.setProperty("zeppelin.interpreter.max.poolsize",
conf.getInt(ZEPPELIN_INTERPRETER_MAX_POOL_SIZE) + "");
}
String interpreterLocalRepoPath = conf.getInterpreterLocalRepoPath();
//TODO(zjffdu) change it to interpreterDir/{interpreter_name}
jProperties.setProperty("zeppelin.interpreter.localRepo",
interpreterLocalRepoPath + "/" + id);
return jProperties;
}
public ZeppelinConfiguration getConf() {
return conf;
}
public void setConf(ZeppelinConfiguration conf) {
this.conf = conf;
}
public List<Dependency> getDependencies() {
if (dependencies == null) {
return new LinkedList<>();
}
return dependencies;
}
public void setDependencies(List<Dependency> dependencies) {
this.dependencies = dependencies;
loadInterpreterDependencies();
}
public InterpreterOption getOption() {
if (option == null) {
option = new InterpreterOption();
}
return option;
}
@ -314,35 +497,32 @@ public class InterpreterSetting {
this.option = option;
}
public String getPath() {
return path;
public String getInterpreterDir() {
return interpreterDir;
}
public void setPath(String path) {
this.path = path;
public void setInterpreterDir(String interpreterDir) {
this.interpreterDir = interpreterDir;
}
public List<InterpreterInfo> getInterpreterInfos() {
return interpreterInfos;
}
void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
this.interpreterGroupFactory = interpreterGroupFactory;
}
void appendDependencies(List<Dependency> dependencies) {
for (Dependency dependency : dependencies) {
if (!this.dependencies.contains(dependency)) {
this.dependencies.add(dependency);
}
}
loadInterpreterDependencies();
}
void setInterpreterOption(InterpreterOption interpreterOption) {
this.option = interpreterOption;
}
public void setProperties(Map<String, InterpreterProperty> p) {
public void setProperties(Properties p) {
this.properties = p;
}
@ -379,6 +559,10 @@ public class InterpreterSetting {
this.errorReason = errorReason;
}
public void setInterpreterInfos(List<InterpreterInfo> interpreterInfos) {
this.interpreterInfos = interpreterInfos;
}
public void setInfos(Map<String, String> infos) {
this.infos = infos;
}
@ -415,7 +599,236 @@ public class InterpreterSetting {
runtimeInfosToBeCleared = null;
}
// For backward compatibility of interpreter.json format after ZEPPELIN-2654
//////////////////////////// IMPORTANT ////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
// This is the only place to create interpreters. For now we always create multiple interpreter
// together (one session). We don't support to create single interpreter yet.
List<Interpreter> createInterpreters(String user, String sessionId) {
List<Interpreter> interpreters = new ArrayList<>();
List<InterpreterInfo> interpreterInfos = getInterpreterInfos();
for (InterpreterInfo info : interpreterInfos) {
Interpreter interpreter = null;
if (option.isRemote()) {
interpreter = new RemoteInterpreter(getJavaProperties(), sessionId,
info.getClassName(), user);
} else {
interpreter = createLocalInterpreter(info.getClassName());
}
if (info.isDefaultInterpreter()) {
interpreters.add(0, interpreter);
} else {
interpreters.add(interpreter);
}
LOGGER.info("Interpreter {} created for user: {}, sessionId: {}",
interpreter.getClassName(), user, sessionId);
}
return interpreters;
}
// Create Interpreter in ZeppelinServer for non-remote mode
private Interpreter createLocalInterpreter(String className)
throws InterpreterException {
LOGGER.info("Create Local Interpreter {} from {}", className, interpreterDir);
ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
try {
URLClassLoader ccl = cleanCl.get(interpreterDir);
if (ccl == null) {
// classloader fallback
ccl = URLClassLoader.newInstance(new URL[]{}, oldcl);
}
boolean separateCL = true;
try { // check if server's classloader has driver already.
Class cls = this.getClass().forName(className);
if (cls != null) {
separateCL = false;
}
} catch (Exception e) {
LOGGER.error("exception checking server classloader driver", e);
}
URLClassLoader cl;
if (separateCL == true) {
cl = URLClassLoader.newInstance(new URL[]{}, ccl);
} else {
cl = ccl;
}
Thread.currentThread().setContextClassLoader(cl);
Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className);
Constructor<Interpreter> constructor =
replClass.getConstructor(new Class[]{Properties.class});
Interpreter repl = constructor.newInstance(getJavaProperties());
repl.setClassloaderUrls(ccl.getURLs());
LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl));
return intp;
} catch (SecurityException e) {
throw new InterpreterException(e);
} catch (NoSuchMethodException e) {
throw new InterpreterException(e);
} catch (IllegalArgumentException e) {
throw new InterpreterException(e);
} catch (InstantiationException e) {
throw new InterpreterException(e);
} catch (IllegalAccessException e) {
throw new InterpreterException(e);
} catch (InvocationTargetException e) {
throw new InterpreterException(e);
} catch (ClassNotFoundException e) {
throw new InterpreterException(e);
} finally {
Thread.currentThread().setContextClassLoader(oldcl);
}
}
RemoteInterpreterProcess createInterpreterProcess() {
RemoteInterpreterProcess remoteInterpreterProcess = null;
int connectTimeout =
conf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + id;
if (option.isExistingProcess()) {
// TODO(zjffdu) remove the existing process approach seems no one is using this.
// use the existing process
remoteInterpreterProcess = new RemoteInterpreterRunningProcess(
connectTimeout,
remoteInterpreterProcessListener,
appEventListener,
option.getHost(),
option.getPort());
} else {
// create new remote process
remoteInterpreterProcess = new RemoteInterpreterManagedProcess(
interpreterRunner != null ? interpreterRunner.getPath() :
conf.getInterpreterRemoteRunnerPath(), interpreterDir, localRepoPath,
getEnvFromInterpreterProperty(getJavaProperties()), connectTimeout,
remoteInterpreterProcessListener, appEventListener, group);
}
return remoteInterpreterProcess;
}
private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
Map<String, String> env = new HashMap<>();
for (Object key : property.keySet()) {
if (RemoteInterpreterUtils.isEnvString((String) key)) {
env.put((String) key, property.getProperty((String) key));
}
}
return env;
}
private List<Interpreter> getOrCreateSession(String user, String noteId) {
ManagedInterpreterGroup interpreterGroup = getOrCreateInterpreterGroup(user, noteId);
Preconditions.checkNotNull(interpreterGroup, "No InterpreterGroup existed for user {}, " +
"noteId {}", user, noteId);
String sessionId = getInterpreterSessionId(user, noteId);
return interpreterGroup.getOrCreateSession(user, sessionId);
}
public Interpreter getDefaultInterpreter(String user, String noteId) {
return getOrCreateSession(user, noteId).get(0);
}
public Interpreter getInterpreter(String user, String noteId, String replName) {
Preconditions.checkNotNull(noteId, "noteId should be not null");
Preconditions.checkNotNull(replName, "replName should be not null");
String className = getInterpreterClassFromInterpreterSetting(replName);
if (className == null) {
return null;
}
List<Interpreter> interpreters = getOrCreateSession(user, noteId);
for (Interpreter interpreter : interpreters) {
if (className.equals(interpreter.getClassName())) {
return interpreter;
}
}
return null;
}
private String getInterpreterClassFromInterpreterSetting(String replName) {
Preconditions.checkNotNull(replName, "replName should be not null");
for (InterpreterInfo info : interpreterInfos) {
String infoName = info.getName();
if (null != info.getName() && replName.equals(infoName)) {
return info.getClassName();
}
}
return null;
}
private ManagedInterpreterGroup createInterpreterGroup(String groupId)
throws InterpreterException {
AngularObjectRegistry angularObjectRegistry;
ManagedInterpreterGroup interpreterGroup = new ManagedInterpreterGroup(groupId, this);
if (option.isRemote()) {
angularObjectRegistry =
new RemoteAngularObjectRegistry(groupId, angularObjectRegistryListener, interpreterGroup);
} else {
angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener);
// TODO(moon) : create distributed resource pool for local interpreters and set
}
interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
return interpreterGroup;
}
private void loadInterpreterDependencies() {
setStatus(Status.DOWNLOADING_DEPENDENCIES);
setErrorReason(null);
Thread t = new Thread() {
public void run() {
try {
// dependencies to prevent library conflict
File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + getId());
if (localRepoDir.exists()) {
try {
FileUtils.forceDelete(localRepoDir);
} catch (FileNotFoundException e) {
LOGGER.info("A file that does not exist cannot be deleted, nothing to worry", e);
}
}
// load dependencies
List<Dependency> deps = getDependencies();
if (deps != null) {
for (Dependency d : deps) {
File destDir = new File(
conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO));
if (d.getExclusions() != null) {
dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
new File(destDir, id));
} else {
dependencyResolver
.load(d.getGroupArtifactVersion(), new File(destDir, id));
}
}
}
setStatus(Status.READY);
setErrorReason(null);
} catch (Exception e) {
LOGGER.error(String.format("Error while downloading repos for interpreter group : %s," +
" go to interpreter setting page click on edit and save it again to make " +
"this interpreter work properly. : %s",
getGroup(), e.getLocalizedMessage()), e);
setErrorReason(e.getLocalizedMessage());
setStatus(Status.ERROR);
}
}
};
t.start();
}
//TODO(zjffdu) ugly code, should not use JsonObject as parameter. not readable
public void convertPermissionsFromUsersToOwners(JsonObject jsonObject) {
if (jsonObject != null) {
JsonObject option = jsonObject.getAsJsonObject("option");
@ -434,26 +847,56 @@ public class InterpreterSetting {
}
// For backward compatibility of interpreter.json format after ZEPPELIN-2403
public void convertFlatPropertiesToPropertiesWithWidgets() {
StringMap newProperties = new StringMap();
static Map<String, InterpreterProperty> convertInterpreterProperties(Object properties) {
if (properties != null && properties instanceof StringMap) {
Map<String, InterpreterProperty> newProperties = new HashMap<>();
StringMap p = (StringMap) properties;
for (Object o : p.entrySet()) {
Map.Entry entry = (Map.Entry) o;
if (!(entry.getValue() instanceof StringMap)) {
StringMap newProperty = new StringMap();
newProperty.put("name", entry.getKey());
newProperty.put("value", entry.getValue());
newProperty.put("type", InterpreterPropertyType.TEXTAREA.getValue());
InterpreterProperty newProperty = new InterpreterProperty(
entry.getKey().toString(),
entry.getValue(),
InterpreterPropertyType.STRING.getValue());
newProperties.put(entry.getKey().toString(), newProperty);
} else {
// already converted
return;
return (Map<String, InterpreterProperty>) properties;
}
}
return newProperties;
this.properties = newProperties;
} else if (properties instanceof Map) {
Map<String, Object> dProperties =
(Map<String, Object>) properties;
Map<String, InterpreterProperty> newProperties = new HashMap<>();
for (String key : dProperties.keySet()) {
Object value = dProperties.get(key);
if (value instanceof InterpreterProperty) {
return (Map<String, InterpreterProperty>) properties;
} else if (value instanceof StringMap) {
StringMap stringMap = (StringMap) value;
InterpreterProperty newProperty = new InterpreterProperty(
key,
stringMap.get("value"),
stringMap.containsKey("type") ? stringMap.get("type").toString() : "string");
newProperties.put(newProperty.getName(), newProperty);
} else if (value instanceof DefaultInterpreterProperty){
DefaultInterpreterProperty dProperty = (DefaultInterpreterProperty) value;
InterpreterProperty property = new InterpreterProperty(
key,
dProperty.getValue(),
dProperty.getType() != null ? dProperty.getType() : "string"
// in case user forget to specify type in interpreter-setting.json
);
newProperties.put(key, property);
} else {
throw new RuntimeException("Can not convert this type of property: " + value.getClass());
}
}
return newProperties;
}
throw new RuntimeException("Can not convert this type: " + properties.getClass());
}
}

1141
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
View file

File diff suppressed because it is too large Load diff

136
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java Normal file
View file

@ -0,0 +1,136 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.List;
/**
* ManagedInterpreterGroup runs under zeppelin server
*/
public class ManagedInterpreterGroup extends InterpreterGroup {
private static final Logger LOGGER = LoggerFactory.getLogger(ManagedInterpreterGroup.class);
private InterpreterSetting interpreterSetting;
private RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
/**
* Create InterpreterGroup with given id and interpreterSetting, used in ZeppelinServer
* @param id
* @param interpreterSetting
*/
ManagedInterpreterGroup(String id, InterpreterSetting interpreterSetting) {
super(id);
this.interpreterSetting = interpreterSetting;
}
public InterpreterSetting getInterpreterSetting() {
return interpreterSetting;
}
public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
if (remoteInterpreterProcess == null) {
LOGGER.info("Create InterperterProcess for InterpreterGroup: " + getId());
remoteInterpreterProcess = interpreterSetting.createInterpreterProcess();
}
return remoteInterpreterProcess;
}
public RemoteInterpreterProcess getRemoteInterpreterProcess() {
return remoteInterpreterProcess;
}
/**
* Close all interpreter instances in this group
*/
public synchronized void close() {
LOGGER.info("Close InterpreterGroup: " + id);
for (String sessionId : sessions.keySet()) {
close(sessionId);
}
}
/**
* Close all interpreter instances in this session
* @param sessionId
*/
public synchronized void close(String sessionId) {
LOGGER.info("Close Session: " + sessionId);
close(sessions.remove(sessionId));
//TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server
if (sessions.isEmpty() && interpreterSetting != null) {
LOGGER.info("Remove this InterpreterGroup {} as all the sessions are closed", id);
interpreterSetting.removeInterpreterGroup(id);
if (remoteInterpreterProcess != null) {
LOGGER.info("Kill RemoteIntetrpreterProcess");
remoteInterpreterProcess.stop();
remoteInterpreterProcess = null;
}
}
}
private void close(Collection<Interpreter> interpreters) {
if (interpreters == null) {
return;
}
for (Interpreter interpreter : interpreters) {
Scheduler scheduler = interpreter.getScheduler();
for (Job job : scheduler.getJobsRunning()) {
job.abort();
job.setStatus(Job.Status.ABORT);
LOGGER.info("Job " + job.getJobName() + " aborted ");
}
for (Job job : scheduler.getJobsWaiting()) {
job.abort();
job.setStatus(Job.Status.ABORT);
LOGGER.info("Job " + job.getJobName() + " aborted ");
}
interpreter.close();
//TODO(zjffdu) move the close of schedule to Interpreter
if (null != scheduler) {
SchedulerFactory.singleton().removeScheduler(scheduler.getName());
}
}
}
public synchronized List<Interpreter> getOrCreateSession(String user, String sessionId) {
if (sessions.containsKey(sessionId)) {
return sessions.get(sessionId);
} else {
List<Interpreter> interpreters = interpreterSetting.createInterpreters(user, sessionId);
for (Interpreter interpreter : interpreters) {
interpreter.setInterpreterGroup(this);
}
LOGGER.info("Create Session {} in InterpreterGroup {} for user {}", sessionId, id, user);
sessions.put(sessionId, interpreters);
return interpreters;
}
}
}

4
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
View file

@ -17,19 +17,17 @@
package org.apache.zeppelin.interpreter.install;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Logger;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.util.Util;
import org.sonatype.aether.RepositoryException;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

0
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputBuffer.java
View file

6
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunner.java
View file

@ -17,6 +17,9 @@
package org.apache.zeppelin.interpreter.remote;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@ -24,9 +27,6 @@ import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This thread sends paragraph's append-data
* periodically, rather than continously, with

0
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java
View file

0
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/InterpreterContextRunnerPool.java
View file

7
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObject.java
View file

@ -20,17 +20,18 @@ package org.apache.zeppelin.interpreter.remote;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectListener;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
/**
* Proxy for AngularObject that exists in remote interpreter process
*/
public class RemoteAngularObject extends AngularObject {
private transient InterpreterGroup interpreterGroup;
private transient ManagedInterpreterGroup interpreterGroup;
RemoteAngularObject(String name, Object o, String noteId, String paragraphId,
InterpreterGroup interpreterGroup,
AngularObjectListener listener) {
ManagedInterpreterGroup interpreterGroup,
AngularObjectListener listener) {
super(name, o, noteId, paragraphId, listener);
this.interpreterGroup = interpreterGroup;
}

83
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
View file

@ -17,29 +17,28 @@
package org.apache.zeppelin.interpreter.remote;
import java.util.List;
import org.apache.thrift.TException;
import com.google.gson.Gson;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import java.util.List;
/**
* Proxy for AngularObjectRegistry that exists in remote interpreter process
*/
public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
Logger logger = LoggerFactory.getLogger(RemoteAngularObjectRegistry.class);
private InterpreterGroup interpreterGroup;
private ManagedInterpreterGroup interpreterGroup;
public RemoteAngularObjectRegistry(String interpreterId,
AngularObjectRegistryListener listener,
InterpreterGroup interpreterGroup) {
AngularObjectRegistryListener listener,
ManagedInterpreterGroup interpreterGroup) {
super(interpreterId, listener);
this.interpreterGroup = interpreterGroup;
}
@ -56,31 +55,29 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* @param noteId
* @return
*/
public AngularObject addAndNotifyRemoteProcess(String name, Object o, String noteId, String
paragraphId) {
Gson gson = new Gson();
public AngularObject addAndNotifyRemoteProcess(final String name,
final Object o,
final String noteId,
final String paragraphId) {
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (!remoteInterpreterProcess.isRunning()) {
return super.add(name, o, noteId, paragraphId, true);
}
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
return super.add(name, o, noteId, paragraphId, true);
} catch (TException e) {
broken = true;
logger.error("Error", e);
} catch (Exception e) {
logger.error("Error", e);
} finally {
if (client != null) {
remoteInterpreterProcess.releaseClient(client, broken);
}
}
return null;
remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
Gson gson = new Gson();
client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
return null;
}
}
);
return super.add(name, o, noteId, paragraphId, true);
}
/**
@ -91,30 +88,24 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* @param paragraphId
* @return
*/
public AngularObject removeAndNotifyRemoteProcess(String name, String noteId, String
paragraphId) {
public AngularObject removeAndNotifyRemoteProcess(final String name,
final String noteId,
final String paragraphId) {
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null || !remoteInterpreterProcess.isRunning()) {
return super.remove(name, noteId, paragraphId);
}
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
client.angularObjectRemove(name, noteId, paragraphId);
return super.remove(name, noteId, paragraphId);
} catch (TException e) {
broken = true;
logger.error("Error", e);
} catch (Exception e) {
logger.error("Error", e);
} finally {
if (client != null) {
remoteInterpreterProcess.releaseClient(client, broken);
remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
client.angularObjectRemove(name, noteId, paragraphId);
return null;
}
}
}
return null;
);
return super.remove(name, noteId, paragraphId);
}
public void removeAllAndNotifyRemoteProcess(String noteId, String paragraphId) {

667
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -17,160 +17,68 @@
package org.apache.zeppelin.interpreter.remote;
import java.util.*;
import org.apache.commons.lang3.StringUtils;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.thrift.TException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.RemoteScheduler;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Proxy for Interpreter instance that runs on separate process
*/
public class RemoteInterpreter extends Interpreter {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class);
private static final Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreter.class);
private static final Gson gson = new Gson();
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private final ApplicationEventListener applicationEventListener;
private Gson gson = new Gson();
private String interpreterRunner;
private String interpreterPath;
private String localRepoPath;
private String className;
private String sessionKey;
private FormType formType;
private boolean initialized;
private Map<String, String> env;
private int connectTimeout;
private int maxPoolSize;
private String host;
private int port;
private String sessionId;
private String userName;
private Boolean isUserImpersonate;
private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
private String interpreterGroupName;
private FormType formType;
private RemoteInterpreterProcess interpreterProcess;
private volatile boolean isOpened = false;
private volatile boolean isCreated = false;
/**
* Remote interpreter and manage interpreter process
*/
public RemoteInterpreter(Properties property, String sessionKey, String className,
String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout,
int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
int outputLimit, String interpreterGroupName) {
super(property);
this.sessionKey = sessionKey;
public RemoteInterpreter(Properties properties,
String sessionId,
String className,
String userName) {
super(properties);
this.sessionId = sessionId;
this.className = className;
initialized = false;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.localRepoPath = localRepoPath;
env = getEnvFromInterpreterProperty(property);
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.applicationEventListener = appListener;
this.userName = userName;
this.isUserImpersonate = isUserImpersonate;
this.outputLimit = outputLimit;
this.interpreterGroupName = interpreterGroupName;
}
/**
* Connect to existing process
*/
public RemoteInterpreter(Properties property, String sessionKey, String className, String host,
int port, String localRepoPath, int connectTimeout, int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
int outputLimit) {
super(property);
this.sessionKey = sessionKey;
this.className = className;
initialized = false;
this.host = host;
this.port = port;
this.localRepoPath = localRepoPath;
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.applicationEventListener = appListener;
this.userName = userName;
this.isUserImpersonate = isUserImpersonate;
this.outputLimit = outputLimit;
}
// VisibleForTesting
public RemoteInterpreter(Properties property, String sessionKey, String className,
String interpreterRunner, String interpreterPath, String localRepoPath,
Map<String, String> env, int connectTimeout,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate) {
super(property);
this.className = className;
this.sessionKey = sessionKey;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.localRepoPath = localRepoPath;
env.putAll(getEnvFromInterpreterProperty(property));
this.env = env;
this.connectTimeout = connectTimeout;
this.maxPoolSize = 10;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.applicationEventListener = appListener;
this.userName = userName;
this.isUserImpersonate = isUserImpersonate;
}
private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
Map<String, String> env = new HashMap<String, String>();
StringBuilder sparkConfBuilder = new StringBuilder();
for (String key : property.stringPropertyNames()) {
if (RemoteInterpreterUtils.isEnvString(key)) {
env.put(key, property.getProperty(key));
}
if (key.equals("master")) {
sparkConfBuilder.append(" --master " + property.getProperty("master"));
}
if (isSparkConf(key, property.getProperty(key))) {
sparkConfBuilder.append(" --conf " + key + "=" +
toShellFormat(property.getProperty(key)));
}
}
env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());
return env;
}
private String toShellFormat(String value) {
if (value.contains("\'") && value.contains("\"")) {
throw new RuntimeException("Spark property value could not contain both \" and '");
} else if (value.contains("\'")) {
return "\"" + value + "\"";
} else {
return "\'" + value + "\'";
}
}
static boolean isSparkConf(String key, String value) {
return !StringUtils.isEmpty(key) && key.startsWith("spark.") && !StringUtils.isEmpty(value);
public boolean isOpened() {
return isOpened;
}
@Override
@ -178,202 +86,113 @@ public class RemoteInterpreter extends Interpreter {
return className;
}
private boolean connectToExistingProcess() {
return host != null && port > 0;
public String getSessionId() {
return this.sessionId;
}
public RemoteInterpreterProcess getInterpreterProcess() {
InterpreterGroup intpGroup = getInterpreterGroup();
if (intpGroup == null) {
return null;
public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
if (this.interpreterProcess != null) {
return this.interpreterProcess;
}
synchronized (intpGroup) {
if (intpGroup.getRemoteInterpreterProcess() == null) {
RemoteInterpreterProcess remoteProcess;
if (connectToExistingProcess()) {
remoteProcess = new RemoteInterpreterRunningProcess(
connectTimeout,
remoteInterpreterProcessListener,
applicationEventListener,
host,
port);
} else {
// create new remote process
remoteProcess = new RemoteInterpreterManagedProcess(
interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName);
}
intpGroup.setRemoteInterpreterProcess(remoteProcess);
}
return intpGroup.getRemoteInterpreterProcess();
}
}
public synchronized void init() {
if (initialized == true) {
return;
}
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
final InterpreterGroup interpreterGroup = getInterpreterGroup();
interpreterProcess.setMaxPoolSize(
Math.max(this.maxPoolSize, interpreterProcess.getMaxPoolSize()));
String groupId = interpreterGroup.getId();
ManagedInterpreterGroup intpGroup = getInterpreterGroup();
this.interpreterProcess = intpGroup.getOrCreateInterpreterProcess();
synchronized (interpreterProcess) {
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
logger.info("Create remote interpreter {}", getClassName());
if (localRepoPath != null) {
property.put("zeppelin.interpreter.localRepo", localRepoPath);
}
property.put("zeppelin.interpreter.output.limit", Integer.toString(outputLimit));
client.createInterpreter(groupId, sessionKey,
getClassName(), (Map) property, userName);
// Push angular object loaded from JSON file to remote interpreter
if (!interpreterGroup.isAngularRegistryPushed()) {
pushAngularObjectRegistryToRemote(client);
interpreterGroup.setAngularRegistryPushed(true);
}
} catch (TException e) {
logger.error("Failed to create interpreter: {}", getClassName());
throw new InterpreterException(e);
} finally {
// TODO(jongyoul): Fixed it when not all of interpreter in same interpreter group are broken
interpreterProcess.releaseClient(client, broken);
if (!interpreterProcess.isRunning()) {
interpreterProcess.start(userName, false);
interpreterProcess.getRemoteInterpreterEventPoller()
.setInterpreterProcess(interpreterProcess);
interpreterProcess.getRemoteInterpreterEventPoller().setInterpreterGroup(intpGroup);
interpreterProcess.getRemoteInterpreterEventPoller().start();
}
}
initialized = true;
return interpreterProcess;
}
public ManagedInterpreterGroup getInterpreterGroup() {
return (ManagedInterpreterGroup) super.getInterpreterGroup();
}
@Override
public void open() {
InterpreterGroup interpreterGroup = getInterpreterGroup();
synchronized (this) {
if (!isOpened) {
// create all the interpreters of the same session first, then Open the internal interpreter
// of this RemoteInterpreter.
// The why we we create all the interpreter of the session is because some interpreter
// depends on other interpreter. e.g. PySparkInterpreter depends on SparkInterpreter.
// also see method Interpreter.getInterpreterInTheSameSessionByClassName
for (Interpreter interpreter : getInterpreterGroup()
.getOrCreateSession(userName, sessionId)) {
((RemoteInterpreter) interpreter).internal_create();
}
synchronized (interpreterGroup) {
// initialize all interpreters in this interpreter group
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
// TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
// initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
// doesn't call open method, it's not open. It causes problem while running intp.close()
// In case of Spark, this method initializes all of interpreters and init() method increases
// reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
// other interpreters doesn't do anything because those LazyInterpreters aren't open.
// But for now, we have to initialise all of interpreters for some reasons.
// See Interpreter.getInterpreterInTheSameSessionByClassName(String)
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
if (!initialized) {
// reference per session
interpreterProcess.reference(interpreterGroup, userName, isUserImpersonate);
}
for (Interpreter intp : new ArrayList<>(interpreters)) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
try {
((RemoteInterpreter) p).init();
} catch (InterpreterException e) {
logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
p.getClassName());
interpreters.remove(p);
}
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
LOGGER.info("Open RemoteInterpreter {}", getClassName());
// open interpreter here instead of in the jobRun method in RemoteInterpreterServer
// client.open(sessionId, className);
// Push angular object loaded from JSON file to remote interpreter
synchronized (getInterpreterGroup()) {
if (!getInterpreterGroup().isAngularRegistryPushed()) {
pushAngularObjectRegistryToRemote(client);
getInterpreterGroup().setAngularRegistryPushed(true);
}
}
return null;
}
});
isOpened = true;
}
}
}
private void internal_create() {
synchronized (this) {
if (!isCreated) {
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
LOGGER.info("Create RemoteInterpreter {}", getClassName());
client.createInterpreter(getInterpreterGroup().getId(), sessionId,
className, (Map) property, userName);
return null;
}
});
isCreated = true;
}
}
}
@Override
public void close() {
InterpreterGroup interpreterGroup = getInterpreterGroup();
synchronized (interpreterGroup) {
// close all interpreters in this session
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
// TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
// initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
// doesn't call open method, it's not open. It causes problem while running intp.close()
// In case of Spark, this method initializes all of interpreters and init() method increases
// reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
// other interpreters doesn't do anything because those LazyInterpreters aren't open.
// But for now, we have to initialise all of interpreters for some reasons.
// See Interpreter.getInterpreterInTheSameSessionByClassName(String)
if (initialized) {
// dereference per session
getInterpreterProcess().dereference();
}
for (Interpreter intp : new ArrayList<>(interpreters)) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
if (isOpened) {
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
client.close(sessionId, className);
return null;
}
try {
((RemoteInterpreter) p).closeInterpreter();
} catch (InterpreterException e) {
logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
p.getClassName());
interpreters.remove(p);
}
}
}
}
public void closeInterpreter() {
if (this.initialized == false) {
return;
}
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
if (client != null) {
client.close(sessionKey, className);
}
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} catch (Exception e1) {
throw new InterpreterException(e1);
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
this.initialized = false;
});
isOpened = false;
} else {
LOGGER.warn("close is called when RemoterInterpreter is not opened for " + className);
}
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
public InterpreterResult interpret(final String st, final InterpreterContext context) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("st:\n{}", st);
}
final FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
InterpreterContextRunnerPool interpreterContextRunnerPool = interpreterProcess
.getInterpreterContextRunnerPool();
List<InterpreterContextRunner> runners = context.getRunners();
if (runners != null && runners.size() != 0) {
// assume all runners in this InterpreterContext have the same note id
@ -382,165 +201,153 @@ public class RemoteInterpreter extends Interpreter {
interpreterContextRunnerPool.clear(noteId);
interpreterContextRunnerPool.addAll(noteId, runners);
}
return interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<InterpreterResult>() {
@Override
public InterpreterResult call(Client client) throws Exception {
boolean broken = false;
try {
RemoteInterpreterResult remoteResult = client.interpret(
sessionId, className, st, convert(context));
Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
}.getType());
context.getConfig().clear();
context.getConfig().putAll(remoteConfig);
GUI currentGUI = context.getGui();
if (form == FormType.NATIVE) {
GUI remoteGui = GUI.fromJson(remoteResult.getGui());
currentGUI.clear();
currentGUI.setParams(remoteGui.getParams());
currentGUI.setForms(remoteGui.getForms());
} else if (form == FormType.SIMPLE) {
final Map<String, Input> currentForms = currentGUI.getForms();
final Map<String, Object> currentParams = currentGUI.getParams();
final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
final Map<String, Input> remoteForms = remoteGUI.getForms();
final Map<String, Object> remoteParams = remoteGUI.getParams();
currentForms.putAll(remoteForms);
currentParams.putAll(remoteParams);
}
final GUI currentGUI = context.getGui();
RemoteInterpreterResult remoteResult = client.interpret(
sessionKey, className, st, convert(context));
InterpreterResult result = convert(remoteResult);
return result;
}
}
);
Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
}.getType());
context.getConfig().clear();
context.getConfig().putAll(remoteConfig);
if (form == FormType.NATIVE) {
GUI remoteGui = GUI.fromJson(remoteResult.getGui());
currentGUI.clear();
currentGUI.setParams(remoteGui.getParams());
currentGUI.setForms(remoteGui.getForms());
} else if (form == FormType.SIMPLE) {
final Map<String, Input> currentForms = currentGUI.getForms();
final Map<String, Object> currentParams = currentGUI.getParams();
final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
final Map<String, Input> remoteForms = remoteGUI.getForms();
final Map<String, Object> remoteParams = remoteGUI.getParams();
currentForms.putAll(remoteForms);
currentParams.putAll(remoteParams);
}
InterpreterResult result = convert(remoteResult);
return result;
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
}
@Override
public void cancel(InterpreterContext context) {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
client.cancel(sessionKey, className, convert(context));
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
public void cancel(final InterpreterContext context) {
if (!isOpened) {
LOGGER.warn("Cancel is called when RemoterInterpreter is not opened for " + className);
return;
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
client.cancel(sessionId, className, convert(context));
return null;
}
});
}
@Override
public FormType getFormType() {
open();
if (formType != null) {
return formType;
}
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
formType = FormType.valueOf(client.getFormType(sessionKey, className));
return formType;
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
// it is possible to call getFormType before it is opened
synchronized (this) {
if (!isOpened) {
open();
}
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
FormType type = interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<FormType>() {
@Override
public FormType call(Client client) throws Exception {
formType = FormType.valueOf(client.getFormType(sessionId, className));
return formType;
}
});
return type;
}
@Override
public int getProgress(InterpreterContext context) {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
if (interpreterProcess == null || !interpreterProcess.isRunning()) {
public int getProgress(final InterpreterContext context) {
if (!isOpened) {
LOGGER.warn("getProgress is called when RemoterInterpreter is not opened for " + className);
return 0;
}
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
return client.getProgress(sessionKey, className, convert(context));
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
return interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Integer>() {
@Override
public Integer call(Client client) throws Exception {
return client.getProgress(sessionId, className, convert(context));
}
});
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
List completion = client.completion(sessionKey, className, buf, cursor,
convert(interpreterContext));
return completion;
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
public List<InterpreterCompletion> completion(final String buf, final int cursor,
final InterpreterContext interpreterContext) {
if (!isOpened) {
LOGGER.warn("completion is called when RemoterInterpreter is not opened for " + className);
return new ArrayList<>();
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
return interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<List<InterpreterCompletion>>() {
@Override
public List<InterpreterCompletion> call(Client client) throws Exception {
return client.completion(sessionId, className, buf, cursor,
convert(interpreterContext));
}
});
}
public String getStatus(final String jobId) {
if (!isOpened) {
LOGGER.warn("getStatus is called when RemoteInterpreter is not opened for " + className);
return Job.Status.UNKNOWN.name();
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
return interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<String>() {
@Override
public String call(Client client) throws Exception {
return client.getStatus(sessionId, jobId);
}
});
}
//TODO(zjffdu) Share the Scheduler in the same session or in the same InterpreterGroup ?
@Override
public Scheduler getScheduler() {
int maxConcurrency = maxPoolSize;
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
if (interpreterProcess == null) {
return null;
} else {
return SchedulerFactory.singleton().createOrGetRemoteScheduler(
RemoteInterpreter.class.getName() + sessionKey + interpreterProcess.hashCode(),
sessionKey, interpreterProcess, maxConcurrency);
}
}
int maxConcurrency = Integer.parseInt(
property.getProperty("zeppelin.interpreter.max.poolsize",
ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE.getIntValue() + ""));
private String getInterpreterGroupKey(InterpreterGroup interpreterGroup) {
return interpreterGroup.getId();
Scheduler s = new RemoteScheduler(
RemoteInterpreter.class.getName() + "-" + sessionId,
SchedulerFactory.singleton().getExecutor(),
sessionId,
this,
SchedulerFactory.singleton(),
maxConcurrency);
return SchedulerFactory.singleton().createOrGetScheduler(s);
}
private RemoteInterpreterContext convert(InterpreterContext ic) {
return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(),
ic.getParagraphTitle(), ic.getParagraphText(), ic.getAuthenticationInfo().toJson(),
gson.toJson(ic.getConfig()), ic.getGui().toJson(), gson.toJson(ic.getRunners()));
ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getAuthenticationInfo()),
gson.toJson(ic.getConfig()), gson.toJson(ic.getGui()), gson.toJson(ic.getRunners()));
}
private InterpreterResult convert(RemoteInterpreterResult result) {
@ -557,41 +364,25 @@ public class RemoteInterpreter extends Interpreter {
/**
* Push local angular object registry to
* remote interpreter. This method should be
* call ONLY inside the init() method
* call ONLY once when the first Interpreter is created
*/
void pushAngularObjectRegistryToRemote(Client client) throws TException {
private void pushAngularObjectRegistryToRemote(Client client) throws TException {
final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup()
.getAngularObjectRegistry();
if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) {
final Map<String, Map<String, AngularObject>> registry = angularObjectRegistry
.getRegistry();
logger.info("Push local angular object registry from ZeppelinServer to" +
LOGGER.info("Push local angular object registry from ZeppelinServer to" +
" remote interpreter group {}", this.getInterpreterGroup().getId());
final java.lang.reflect.Type registryType = new TypeToken<Map<String,
Map<String, AngularObject>>>() {
}.getType();
Gson gson = new Gson();
client.angularRegistryPush(gson.toJson(registry, registryType));
}
}
public Map<String, String> getEnv() {
return env;
}
public void addEnv(Map<String, String> env) {
if (this.env == null) {
this.env = new HashMap<>();
}
this.env.putAll(env);
}
//Only for test
public String getInterpreterRunner() {
return interpreterRunner;
@Override
public String toString() {
return "RemoteInterpreter_" + className + "_" + sessionId;
}
}

286
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -26,6 +26,7 @@ import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
@ -38,8 +39,6 @@ import org.apache.zeppelin.resource.ResourceSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;
@ -62,7 +61,7 @@ public class RemoteInterpreterEventPoller extends Thread {
private volatile boolean shutdown;
private RemoteInterpreterProcess interpreterProcess;
private InterpreterGroup interpreterGroup;
private ManagedInterpreterGroup interpreterGroup;
Gson gson = new Gson();
@ -78,13 +77,12 @@ public class RemoteInterpreterEventPoller extends Thread {
this.interpreterProcess = interpreterProcess;
}
public void setInterpreterGroup(InterpreterGroup interpreterGroup) {
public void setInterpreterGroup(ManagedInterpreterGroup interpreterGroup) {
this.interpreterGroup = interpreterGroup;
}
@Override
public void run() {
Client client = null;
AppendOutputRunner runner = new AppendOutputRunner(listener);
ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(
runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
@ -100,26 +98,14 @@ public class RemoteInterpreterEventPoller extends Thread {
continue;
}
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
logger.error("Can't get RemoteInterpreterEvent", e1);
waitQuietly();
continue;
}
RemoteInterpreterEvent event = null;
boolean broken = false;
try {
event = client.getEvent();
} catch (TException e) {
broken = true;
logger.error("Can't get RemoteInterpreterEvent", e);
waitQuietly();
continue;
} finally {
interpreterProcess.releaseClient(client, broken);
}
RemoteInterpreterEvent event = interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<RemoteInterpreterEvent>() {
@Override
public RemoteInterpreterEvent call(Client client) throws Exception {
return client.getEvent();
}
}
);
AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
@ -286,10 +272,7 @@ public class RemoteInterpreterEventPoller extends Thread {
boolean broken = false;
final Gson gson = new Gson();
final String eventOwnerKey = reqResourceBody.getOwnerKey();
Client interpreterServerMain = null;
try {
interpreterServerMain = interpreterProcess.getClient();
final Client eventClient = interpreterServerMain;
if (resourceType == RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS) {
final List<ZeppelinServerResourceParagraphRunner> remoteRunners = new LinkedList<>();
@ -308,7 +291,6 @@ public class RemoteInterpreterEventPoller extends Thread {
@Override
public void onFinished(Object resultObject) {
boolean clientBroken = false;
if (resultObject != null && resultObject instanceof List) {
List<InterpreterContextRunner> runnerList =
(List<InterpreterContextRunner>) resultObject;
@ -324,15 +306,15 @@ public class RemoteInterpreterEventPoller extends Thread {
resResource.setResourceType(RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS);
resResource.setData(remoteRunners);
try {
eventClient.onReceivedZeppelinResource(resResource.toJson());
} catch (Exception e) {
clientBroken = true;
logger.error("Can't get RemoteInterpreterEvent", e);
waitQuietly();
} finally {
interpreterProcess.releaseClient(eventClient, clientBroken);
}
interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
client.onReceivedZeppelinResource(resResource.toJson());
return null;
}
}
);
}
}
@ -346,39 +328,32 @@ public class RemoteInterpreterEventPoller extends Thread {
reqRunnerContext.getNoteId(), reqRunnerContext.getParagraphId(), callBackEvent);
}
} catch (Exception e) {
broken = true;
logger.error("Can't get RemoteInterpreterEvent", e);
waitQuietly();
} finally {
interpreterProcess.releaseClient(interpreterServerMain, broken);
}
}
private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) {
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
List<String> resourceList = new LinkedList<>();
Gson gson = new Gson();
for (Resource r : resourceSet) {
resourceList.add(gson.toJson(r));
}
client.resourcePoolResponseGetAll(resourceList);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
}
private void sendResourcePoolResponseGetAll(final ResourceSet resourceSet) {
interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
List<String> resourceList = new LinkedList<>();
for (Resource r : resourceSet) {
resourceList.add(r.toJson());
}
client.resourcePoolResponseGetAll(resourceList);
return null;
}
}
);
}
private ResourceSet getAllResourcePoolExcept() {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
for (ManagedInterpreterGroup intpGroup : interpreterGroup.getInterpreterSetting()
.getInterpreterSettingManager().getAllInterpreterGroup()) {
if (intpGroup.getId().equals(interpreterGroup.getId())) {
continue;
}
@ -390,115 +365,94 @@ public class RemoteInterpreterEventPoller extends Thread {
resourceSet.addAll(localPool.getAll());
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
List<String> resourceList = client.resourcePoolGetAll();
Gson gson = new Gson();
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
@Override
public List<String> call(Client client) throws Exception {
return client.resourcePoolGetAll();
}
}
);
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
}
}
return resourceSet;
}
private void sendResourceResponseGet(ResourceId resourceId, Object o) {
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
Gson gson = new Gson();
String rid = gson.toJson(resourceId);
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseGet(rid, obj);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
}
private void sendResourceResponseGet(final ResourceId resourceId, final Object o) {
interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
String rid = resourceId.toJson();
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseGet(rid, obj);
return null;
}
}
);
}
private Object getResource(ResourceId resourceId) {
InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
resourceId.getResourcePoolId());
private Object getResource(final ResourceId resourceId) {
ManagedInterpreterGroup intpGroup = interpreterGroup.getInterpreterSetting()
.getInterpreterSettingManager()
.getInterpreterGroupById(resourceId.getResourcePoolId());
if (intpGroup == null) {
return null;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
return localPool.get(resourceId.getName());
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
ByteBuffer res = client.resourceGet(
resourceId.getNoteId(),
resourceId.getParagraphId(),
resourceId.getName());
Object o = Resource.deserializeObject(res);
return o;
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
ByteBuffer buffer = remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<ByteBuffer>() {
@Override
public ByteBuffer call(Client client) throws Exception {
return client.resourceGet(
resourceId.getNoteId(),
resourceId.getParagraphId(),
resourceId.getName());
}
}
}
);
try {
Object o = Resource.deserializeObject(buffer);
return o;
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return null;
}
public void sendInvokeMethodResult(InvokeResourceMethodEventMessage message, Object o) {
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
Gson gson = new Gson();
String invokeMessage = gson.toJson(message);
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseInvokeMethod(invokeMessage, obj);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
}
public void sendInvokeMethodResult(final InvokeResourceMethodEventMessage message,
final Object o) {
interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
String invokeMessage = message.toJson();
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseInvokeMethod(invokeMessage, obj);
return null;
}
}
);
}
private Object invokeResourceMethod(InvokeResourceMethodEventMessage message) {
ResourceId resourceId = message.resourceId;
InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
resourceId.getResourcePoolId());
private Object invokeResourceMethod(final InvokeResourceMethodEventMessage message) {
final ResourceId resourceId = message.resourceId;
ManagedInterpreterGroup intpGroup = interpreterGroup.getInterpreterSetting()
.getInterpreterSettingManager().getInterpreterGroupById(resourceId.getResourcePoolId());
if (intpGroup == null) {
return null;
}
@ -529,25 +483,25 @@ public class RemoteInterpreterEventPoller extends Thread {
return null;
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
ByteBuffer res = interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<ByteBuffer>() {
@Override
public ByteBuffer call(Client client) throws Exception {
return client.resourceInvokeMethod(
resourceId.getNoteId(),
resourceId.getParagraphId(),
resourceId.getName(),
message.toJson());
}
}
);
try {
client = remoteInterpreterProcess.getClient();
ByteBuffer res = client.resourceInvokeMethod(
resourceId.getNoteId(),
resourceId.getParagraphId(),
resourceId.getName(),
gson.toJson(message));
Object o = Resource.deserializeObject(res);
return o;
return Resource.deserializeObject(res);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
return null;
}
return null;
}

13
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
View file

@ -21,6 +21,7 @@ import org.apache.commons.exec.*;
import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -97,6 +98,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
// start server process
try {
port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
logger.info("Choose port {} for RemoteInterpreterProcess", port);
} catch (IOException e1) {
throw new InterpreterException(e1);
}
@ -172,6 +174,17 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
public void stop() {
if (isRunning()) {
logger.info("kill interpreter process");
try {
callRemoteFunction(new RemoteFunction<Void>() {
@Override
public Void call(RemoteInterpreterService.Client client) throws Exception {
client.shutdown();
return null;
}
});
} catch (Exception e) {
logger.warn("ignore the exception when shutting down");
}
watchdog.destroyProcess();
}

154
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
View file

@ -20,11 +20,10 @@ import com.google.gson.Gson;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.thrift.TException;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Abstract class for interpreter process
@ -32,9 +31,6 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class RemoteInterpreterProcess {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class);
// number of sessions that are attached to this process
private final AtomicInteger referenceCount;
private GenericObjectPool<Client> clientPool;
private final RemoteInterpreterEventPoller remoteInterpreterEventPoller;
private final InterpreterContextRunnerPool interpreterContextRunnerPool;
@ -46,16 +42,20 @@ public abstract class RemoteInterpreterProcess {
ApplicationEventListener appListener) {
this(new RemoteInterpreterEventPoller(listener, appListener),
connectTimeout);
this.remoteInterpreterEventPoller.setInterpreterProcess(this);
}
RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller,
int connectTimeout) {
this.interpreterContextRunnerPool = new InterpreterContextRunnerPool();
referenceCount = new AtomicInteger(0);
this.remoteInterpreterEventPoller = remoteInterpreterEventPoller;
this.connectTimeout = connectTimeout;
}
public RemoteInterpreterEventPoller getRemoteInterpreterEventPoller() {
return remoteInterpreterEventPoller;
}
public abstract String getHost();
public abstract int getPort();
public abstract void start(String userName, Boolean isUserImpersonate);
@ -66,37 +66,18 @@ public abstract class RemoteInterpreterProcess {
return connectTimeout;
}
public int reference(InterpreterGroup interpreterGroup, String userName,
Boolean isUserImpersonate) {
synchronized (referenceCount) {
if (!isRunning()) {
start(userName, isUserImpersonate);
}
if (clientPool == null) {
clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
clientPool.setTestOnBorrow(true);
remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup);
remoteInterpreterEventPoller.setInterpreterProcess(this);
remoteInterpreterEventPoller.start();
}
return referenceCount.incrementAndGet();
}
}
public Client getClient() throws Exception {
public synchronized Client getClient() throws Exception {
if (clientPool == null || clientPool.isClosed()) {
return null;
clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
}
return clientPool.borrowObject();
}
public void releaseClient(Client client) {
private void releaseClient(Client client) {
releaseClient(client, false);
}
public void releaseClient(Client client, boolean broken) {
private void releaseClient(Client client, boolean broken) {
if (broken) {
releaseBrokenClient(client);
} else {
@ -108,7 +89,7 @@ public abstract class RemoteInterpreterProcess {
}
}
public void releaseBrokenClient(Client client) {
private void releaseBrokenClient(Client client) {
try {
clientPool.invalidateObject(client);
} catch (Exception e) {
@ -116,90 +97,6 @@ public abstract class RemoteInterpreterProcess {
}
}
public int dereference() {
synchronized (referenceCount) {
int r = referenceCount.decrementAndGet();
if (r == 0) {
logger.info("shutdown interpreter process");
remoteInterpreterEventPoller.shutdown();
// first try shutdown
Client client = null;
try {
client = getClient();
client.shutdown();
} catch (Exception e) {
// safely ignore exception while client.shutdown() may terminates remote process
logger.info("Exception in RemoteInterpreterProcess while synchronized dereference, can " +
"safely ignore exception while client.shutdown() may terminates remote process");
logger.debug(e.getMessage(), e);
} finally {
if (client != null) {
// no longer used
releaseBrokenClient(client);
}
}
clientPool.clear();
clientPool.close();
// wait for some time (connectTimeout) and force kill
// remote process server.serve() loop is not always finishing gracefully
long startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < connectTimeout) {
if (this.isRunning()) {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
logger.error("Exception in RemoteInterpreterProcess while synchronized dereference " +
"Thread.sleep", e);
}
} else {
break;
}
}
}
return r;
}
}
public int referenceCount() {
synchronized (referenceCount) {
return referenceCount.get();
}
}
public int getNumActiveClient() {
if (clientPool == null) {
return 0;
} else {
return clientPool.getNumActive();
}
}
public int getNumIdleClient() {
if (clientPool == null) {
return 0;
} else {
return clientPool.getNumIdle();
}
}
public void setMaxPoolSize(int size) {
if (clientPool != null) {
//Size + 2 for progress poller , cancel operation
clientPool.setMaxTotal(size + 2);
}
}
public int getMaxPoolSize() {
if (clientPool != null) {
return clientPool.getMaxTotal();
} else {
return 0;
}
}
/**
* Called when angular object is updated in client side to propagate
* change to the remote process
@ -239,4 +136,33 @@ public abstract class RemoteInterpreterProcess {
public InterpreterContextRunnerPool getInterpreterContextRunnerPool() {
return interpreterContextRunnerPool;
}
public <T> T callRemoteFunction(RemoteFunction<T> func) {
Client client = null;
boolean broken = false;
try {
client = getClient();
if (client != null) {
return func.call(client);
}
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} catch (Exception e1) {
throw new InterpreterException(e1);
} finally {
if (client != null) {
releaseClient(client, broken);
}
}
return null;
}
/**
*
* @param <T>
*/
public interface RemoteFunction<T> {
T call(Client client) throws Exception;
}
}

2
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessListener.java
View file

@ -41,5 +41,5 @@ public interface RemoteInterpreterProcessListener {
public void onError();
}
public void onParaInfosReceived(String noteId, String paragraphId,
String interpreterSettingId, Map<String, String> metaInfos);
String interpreterSettingId, Map<String, String> metaInfos);
}

1
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
View file

@ -17,7 +17,6 @@
package org.apache.zeppelin.notebook;
import org.apache.zeppelin.helium.HeliumPackage;
import org.apache.zeppelin.interpreter.InterpreterGroup;
/**
* Current state of application

15
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -20,7 +20,6 @@ package org.apache.zeppelin.notebook;
import static java.lang.String.format;
import java.io.IOException;
import java.io.Serializable;
import java.util.*;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@ -28,8 +27,6 @@ import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.common.JsonSerializable;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@ -41,7 +38,6 @@ import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.utility.IdHashes;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.search.SearchService;
@ -126,11 +122,6 @@ public class Note implements ParagraphJobListener, JsonSerializable {
id = IdHashes.generateId();
}
private String getDefaultInterpreterName() {
InterpreterSetting setting = interpreterSettingManager.getDefaultInterpreterSetting(getId());
return null != setting ? setting.getName() : StringUtils.EMPTY;
}
public boolean isPersonalizedMode() {
Object v = getConfig().get("personalizedMode");
return null != v && "true".equals(v);
@ -385,7 +376,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
*/
public Paragraph removeParagraph(String user, String paragraphId) {
removeAllAngularObjectInParagraph(user, paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
interpreterSettingManager.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
@ -690,7 +681,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
}
@ -705,7 +696,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {

29
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -18,7 +18,6 @@
package org.apache.zeppelin.notebook;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -35,10 +34,8 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.stream.JsonReader;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.quartz.CronScheduleBuilder;
import org.quartz.CronTrigger;
import org.quartz.JobBuilder;
@ -56,11 +53,9 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
@ -140,7 +135,7 @@ public class Notebook implements NoteEventListener {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
note = createNote(interpreterSettingManager.getDefaultInterpreterSettingList(), subject);
note = createNote(interpreterSettingManager.getInterpreterSettingIds(), subject);
} else {
note = createNote(null, subject);
}
@ -270,8 +265,8 @@ public class Notebook implements NoteEventListener {
}
}
interpreterSettingManager.setInterpreters(user, note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreters(...) do the same
interpreterSettingManager.setInterpreterBinding(user, note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreterBinding(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
}
@ -279,7 +274,7 @@ public class Notebook implements NoteEventListener {
List<String> getBindedInterpreterSettingsIds(String id) {
Note note = getNote(id);
if (note != null) {
return interpreterSettingManager.getInterpreters(note.getId());
return interpreterSettingManager.getInterpreterBinding(note.getId());
} else {
return new LinkedList<>();
}
@ -313,9 +308,10 @@ public class Notebook implements NoteEventListener {
}
public void moveNoteToTrash(String noteId) {
for (InterpreterSetting interpreterSetting : interpreterSettingManager
.getInterpreterSettings(noteId)) {
interpreterSettingManager.removeInterpretersForNote(interpreterSetting, "", noteId);
try {
interpreterSettingManager.setInterpreterBinding("", noteId, new ArrayList<String>());
} catch (IOException e) {
e.printStackTrace();
}
}
@ -339,7 +335,7 @@ public class Notebook implements NoteEventListener {
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : interpreterSettingManager.get()) {
AngularObjectRegistry registry =
settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
settings.getOrCreateInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
@ -374,7 +370,7 @@ public class Notebook implements NoteEventListener {
}
}
ResourcePoolUtils.removeResourcesBelongsToNote(id);
interpreterSettingManager.removeResourcesBelongsToNote(id);
fireNoteRemoveEvent(note);
@ -521,7 +517,8 @@ public class Notebook implements NoteEventListener {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId());
InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(subject.getUser(),
note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();

23
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -28,8 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.common.JsonSerializable;
@ -93,10 +91,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
// since zeppelin-0.7.0, zeppelin stores multiple results of the paragraph
// see ZEPPELIN-212
Object results;
volatile Object results;
// For backward compatibility of note.json format after ZEPPELIN-212
Object result;
volatile Object result;
private Map<String, ParagraphRuntimeInfo> runtimeInfos;
/**
@ -157,7 +155,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
@Override
public void setResult(Object results) {
public synchronized void setResult(Object results) {
this.results = results;
}
@ -354,7 +352,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
@Override
public Object getReturn() {
public synchronized Object getReturn() {
return results;
}
@ -401,6 +399,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
logger.error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
//TODO(zjffdu) check interpreter setting status in interpreter setting itself
InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
while (intp.getStatus().equals(
org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
@ -560,8 +559,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup =
interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
.getAngularObjectRegistry();
resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
.getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
@ -591,8 +592,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup =
interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
.getAngularObjectRegistry();
resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
.getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();

136
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java → zeppelin-zengine/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
View file

@ -17,11 +17,9 @@
package org.apache.zeppelin.scheduler;
import org.apache.thrift.TException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.scheduler.Job.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,6 +32,7 @@ import java.util.concurrent.ExecutorService;
/**
* RemoteScheduler runs in ZeppelinServer and proxies Scheduler running on RemoteInterpreter
*
*/
public class RemoteScheduler implements Scheduler {
Logger logger = LoggerFactory.getLogger(RemoteScheduler.class);
@ -45,17 +44,17 @@ public class RemoteScheduler implements Scheduler {
boolean terminate = false;
private String name;
private int maxConcurrency;
private final String noteId;
private RemoteInterpreterProcess interpreterProcess;
private final String sessionId;
private RemoteInterpreter remoteInterpreter;
public RemoteScheduler(String name, ExecutorService executor, String noteId,
RemoteInterpreterProcess interpreterProcess, SchedulerListener listener,
public RemoteScheduler(String name, ExecutorService executor, String sessionId,
RemoteInterpreter remoteInterpreter, SchedulerListener listener,
int maxConcurrency) {
this.name = name;
this.executor = executor;
this.listener = listener;
this.noteId = noteId;
this.interpreterProcess = interpreterProcess;
this.sessionId = sessionId;
this.remoteInterpreter = remoteInterpreter;
this.maxConcurrency = maxConcurrency;
}
@ -167,14 +166,15 @@ public class RemoteScheduler implements Scheduler {
private long initialPeriodMsec;
private long initialPeriodCheckIntervalMsec;
private long checkIntervalMsec;
private boolean terminate;
private volatile boolean terminate;
private JobListener listener;
private Job job;
Status lastStatus;
volatile Status lastStatus;
public JobStatusPoller(long initialPeriodMsec,
long initialPeriodCheckIntervalMsec, long checkIntervalMsec, Job job,
JobListener listener) {
setName("JobStatusPoller-" + job.getId());
this.initialPeriodMsec = initialPeriodMsec;
this.initialPeriodCheckIntervalMsec = initialPeriodCheckIntervalMsec;
this.checkIntervalMsec = checkIntervalMsec;
@ -209,7 +209,7 @@ public class RemoteScheduler implements Scheduler {
}
Status newStatus = getStatus();
if (newStatus == null) { // unknown
if (newStatus == Status.UNKNOWN) { // unknown
continue;
}
@ -231,7 +231,9 @@ public class RemoteScheduler implements Scheduler {
private Status getLastStatus() {
if (terminate == true) {
if (lastStatus != Status.FINISHED &&
if (job.getErrorMessage() != null) {
return Status.ERROR;
} else if (lastStatus != Status.FINISHED &&
lastStatus != Status.ERROR &&
lastStatus != Status.ABORT) {
return Status.FINISHED;
@ -239,58 +241,35 @@ public class RemoteScheduler implements Scheduler {
return (lastStatus == null) ? Status.FINISHED : lastStatus;
}
} else {
return (lastStatus == null) ? Status.FINISHED : lastStatus;
return (lastStatus == null) ? Status.UNKNOWN : lastStatus;
}
}
public synchronized Job.Status getStatus() {
if (interpreterProcess.referenceCount() <= 0) {
public synchronized Status getStatus() {
if (!remoteInterpreter.isOpened()) {
return getLastStatus();
}
Client client;
try {
client = interpreterProcess.getClient();
} catch (Exception e) {
logger.error("Can't get status information", e);
lastStatus = Status.ERROR;
return Status.ERROR;
}
boolean broken = false;
try {
String statusStr = client.getStatus(noteId, job.getId());
if ("Unknown".equals(statusStr)) {
// not found this job in the remote schedulers.
// maybe not submitted, maybe already finished
//Status status = getLastStatus();
listener.afterStatusChange(job, null, null);
return job.getStatus();
}
Status status = Status.valueOf(statusStr);
lastStatus = status;
listener.afterStatusChange(job, null, status);
return status;
} catch (TException e) {
broken = true;
logger.error("Can't get status information", e);
lastStatus = Status.ERROR;
return Status.ERROR;
} catch (Exception e) {
logger.error("Unknown status", e);
lastStatus = Status.ERROR;
return Status.ERROR;
} finally {
interpreterProcess.releaseClient(client, broken);
Status status = Status.valueOf(remoteInterpreter.getStatus(job.getId()));
if (status == Status.UNKNOWN) {
// not found this job in the remote schedulers.
// maybe not submitted, maybe already finished
//Status status = getLastStatus();
listener.afterStatusChange(job, null, null);
return job.getStatus();
}
lastStatus = status;
listener.afterStatusChange(job, null, status);
return status;
}
}
//TODO(zjffdu) need to refactor the schdule module which is too complicated
private class JobRunner implements Runnable, JobListener {
private final Logger logger = LoggerFactory.getLogger(JobRunner.class);
private Scheduler scheduler;
private Job job;
private boolean jobExecuted;
boolean jobSubmittedRemotely;
private volatile boolean jobExecuted;
volatile boolean jobSubmittedRemotely;
public JobRunner(Scheduler scheduler, Job job) {
this.scheduler = scheduler;
@ -338,20 +317,22 @@ public class RemoteScheduler implements Scheduler {
}
// set job status based on result.
Status lastStatus = jobStatusPoller.getStatus();
Object jobResult = job.getReturn();
if (jobResult != null && jobResult instanceof InterpreterResult) {
if (((InterpreterResult) jobResult).code() == Code.ERROR) {
lastStatus = Status.ERROR;
}
}
if (job.getException() != null) {
lastStatus = Status.ERROR;
if (job.isAborted()) {
job.setStatus(Status.ABORT);
} else if (job.getException() != null) {
// logger.info("Job ABORT, " + job.getId());
job.setStatus(Status.ERROR);
} else if (jobResult != null && jobResult instanceof InterpreterResult
&& ((InterpreterResult) jobResult).code() == Code.ERROR) {
// logger.info("Job Error, " + job.getId());
job.setStatus(Status.ERROR);
} else {
// logger.info("Job Finished, " + job.getId());
job.setStatus(Status.FINISHED);
}
synchronized (queue) {
job.setStatus(lastStatus);
if (listener != null) {
listener.jobFinished(scheduler, job);
}
@ -374,25 +355,6 @@ public class RemoteScheduler implements Scheduler {
@Override
public void afterStatusChange(Job job, Status before, Status after) {
if (after == null) { // unknown. maybe before sumitted remotely, maybe already finished.
if (jobExecuted) {
jobSubmittedRemotely = true;
Object jobResult = job.getReturn();
if (job.isAborted()) {
job.setStatus(Status.ABORT);
} else if (job.getException() != null) {
job.setStatus(Status.ERROR);
} else if (jobResult != null && jobResult instanceof InterpreterResult
&& ((InterpreterResult) jobResult).code() == Code.ERROR) {
job.setStatus(Status.ERROR);
} else {
job.setStatus(Status.FINISHED);
}
}
return;
}
// Update remoteStatus
if (jobExecuted == false) {
if (after == Status.FINISHED || after == Status.ABORT
@ -402,14 +364,18 @@ public class RemoteScheduler implements Scheduler {
return;
} else if (after == Status.RUNNING) {
jobSubmittedRemotely = true;
job.setStatus(Status.RUNNING);
// logger.info("Job RUNNING, " + job.getId());
}
} else {
jobSubmittedRemotely = true;
}
// status polled by status poller
if (job.getStatus() != after) {
job.setStatus(after);
// only set status when it is RUNNING
// We would set other status based on the interpret result
if (after == Status.RUNNING) {
// logger.info("Job RUNNING, " + job.getId());
job.setStatus(Status.RUNNING);
}
}
}

97
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
View file

@ -16,15 +16,18 @@
*/
package org.apache.zeppelin.helium;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.ApplicationState;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.ParagraphJobListener;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@ -35,24 +38,16 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
public class HeliumApplicationFactoryTest implements JobListenerFactory {
private File tmpDir;
private File notebookDir;
private ZeppelinConfiguration conf;
public class HeliumApplicationFactoryTest extends AbstractInterpreterTest implements JobListenerFactory {
private SchedulerFactory schedulerFactory;
private DependencyResolver depResolver;
private InterpreterFactory factory;
private InterpreterSettingManager interpreterSettingManager;
private VFSNotebookRepo notebookRepo;
private Notebook notebook;
private HeliumApplicationFactory heliumAppFactory;
@ -60,46 +55,15 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZepelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
File confDir = new File(tmpDir, "conf");
confDir.mkdirs();
notebookDir = new File(tmpDir + "/notebook");
notebookDir.mkdirs();
File home = new File(getClass().getClassLoader().getResource("note").getFile()) // zeppelin/zeppelin-zengine/target/test-classes/note
.getParentFile() // zeppelin/zeppelin-zengine/target/test-classes
.getParentFile() // zeppelin/zeppelin-zengine/target
.getParentFile() // zeppelin/zeppelin-zengine
.getParentFile(); // zeppelin
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), home.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.getAbsolutePath() + "/conf");
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
conf = new ZeppelinConfiguration();
this.schedulerFactory = new SchedulerFactory();
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
super.setUp();
this.schedulerFactory = SchedulerFactory.singleton();
heliumAppFactory = new HeliumApplicationFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager);
HashMap<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
factory.setEnv(env);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
// set AppEventListener properly
for (InterpreterSetting interpreterSetting : interpreterSettingManager.get()) {
interpreterSetting.setAppEventListener(heliumAppFactory);
}
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
@ -108,7 +72,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
conf,
notebookRepo,
schedulerFactory,
factory,
interpreterFactory,
interpreterSettingManager,
this,
search,
@ -124,16 +88,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@After
public void tearDown() throws Exception {
List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) {
intpGroup.close();
}
}
FileUtils.deleteDirectory(tmpDir);
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(),
ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getStringValue());
super.tearDown();
}
@ -150,7 +105,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note1.getId(),interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note1.getId(),interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -196,7 +151,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -236,7 +191,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -297,7 +252,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {

8
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
View file

@ -52,7 +52,7 @@ public class HeliumTest {
// given
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(),
null, null, null);
null, null, null, null);
assertFalse(heliumConf.exists());
// when
@ -63,14 +63,14 @@ public class HeliumTest {
// then load without exception
Helium heliumRestored = new Helium(
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
}
@Test
public void testRestoreRegistryInstances() throws IOException, URISyntaxException, TaskRunnerException {
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1");
HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", "r2");
helium.addRegistry(registry1);
@ -105,7 +105,7 @@ public class HeliumTest {
public void testRefresh() throws IOException, URISyntaxException, TaskRunnerException {
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1");
helium.addRegistry(registry1);

74
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java Normal file
View file

@ -0,0 +1,74 @@
package org.apache.zeppelin.interpreter;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import static org.mockito.Mockito.mock;
/**
* This class will load configuration files under
* src/test/resources/interpreter
* src/test/resources/conf
*
* to construct InterpreterSettingManager and InterpreterFactory properly
*
*/
public abstract class AbstractInterpreterTest {
protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractInterpreterTest.class);
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
protected InterpreterSettingManager interpreterSettingManager;
protected InterpreterFactory interpreterFactory;
protected File testRootDir;
protected File interpreterDir;
protected File confDir;
protected File notebookDir;
protected ZeppelinConfiguration conf;
@Before
public void setUp() throws Exception {
// copy the resources files to a temp folder
testRootDir = new File(System.getProperty("java.io.tmpdir") + "/Zeppelin_Test_" + System.currentTimeMillis());
testRootDir.mkdirs();
LOGGER.info("Create tmp directory: {} as root folder of ZEPPELIN_INTERPRETER_DIR & ZEPPELIN_CONF_DIR", testRootDir.getAbsolutePath());
interpreterDir = new File(testRootDir, "interpreter");
confDir = new File(testRootDir, "conf");
notebookDir = new File(testRootDir, "notebook");
interpreterDir.mkdirs();
confDir.mkdirs();
notebookDir.mkdirs();
FileUtils.copyDirectory(new File("src/test/resources/interpreter"), interpreterDir);
FileUtils.copyDirectory(new File("src/test/resources/conf"), confDir);
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_DIR.getVarName(), interpreterDir.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(), INTERPRETER_SCRIPT);
conf = new ZeppelinConfiguration();
interpreterSettingManager = new InterpreterSettingManager(conf,
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
interpreterFactory = new InterpreterFactory(interpreterSettingManager);
}
@After
public void tearDown() throws Exception {
interpreterSettingManager.close();
FileUtils.deleteDirectory(testRootDir);
}
}

45
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java → zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java
View file

@ -14,13 +14,46 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.apache.commons.lang.NullArgumentException;
import java.util.Properties;
/**
* Created InterpreterGroup
*/
public interface InterpreterGroupFactory {
InterpreterGroup createInterpreterGroup(String interpreterGroupId, InterpreterOption option);
public class DoubleEchoInterpreter extends Interpreter {
public DoubleEchoInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, st + "," + st);
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return null;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}

65
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java Normal file
View file

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import java.util.Properties;
/**
* Just return the received statement back
*/
public class EchoInterpreter extends Interpreter {
public EchoInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
if (Boolean.parseBoolean(property.getProperty("zeppelin.interpreter.echo.fail", "false"))) {
return new InterpreterResult(InterpreterResult.Code.ERROR);
} else {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, st);
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}

485
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
View file

@ -17,481 +17,50 @@
package org.apache.zeppelin.interpreter;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.NullArgumentException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.quartz.SchedulerException;
import org.sonatype.aether.RepositoryException;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class InterpreterFactoryTest {
public class InterpreterFactoryTest extends AbstractInterpreterTest {
private InterpreterFactory factory;
private InterpreterSettingManager interpreterSettingManager;
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterContext context;
private Notebook notebook;
private NotebookRepo notebookRepo;
private DependencyResolver depResolver;
private SchedulerFactory schedulerFactory;
private NotebookAuthorization notebookAuthorization;
@Mock
private JobListenerFactory jobListenerFactory;
@Test
public void testGetFactory() throws IOException {
// no default interpreter because there's no interpreter setting binded to this note
assertNull(interpreterFactory.getInterpreter("user1", "note1", ""));
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
new File(tmpDir, "conf").mkdirs();
FileUtils.copyDirectory(new File("src/test/resources/interpreter"), new File(tmpDir, "interpreter"));
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "") instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "");
// EchoInterpreter is the default interpreter (see zeppelin-interpreter/src/test/resources/conf/interpreter.json)
assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(),
"mock1,mock2,mock11,dev");
conf = new ZeppelinConfiguration();
schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null);
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "test") instanceof RemoteInterpreter);
remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "test");
assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
intp1Properties.put("PROPERTY_1",
new InterpreterProperty("PROPERTY_1", "VALUE_1"));
intp1Properties.put("property_2",
new InterpreterProperty("property_2", "value_2"));
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "echo") instanceof RemoteInterpreter);
remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "echo");
assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, jobListenerFactory, search,
notebookAuthorization, null);
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "double_echo") instanceof RemoteInterpreter);
remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "double_echo");
assertEquals(DoubleEchoInterpreter.class.getName(), remoteInterpreter.getClassName());
}
@After
public void tearDown() throws Exception {
FileUtils.deleteDirectory(tmpDir);
@Test(expected = InterpreterException.class)
public void testUnknownRepl1() throws IOException {
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl");
}
@Test
public void testBasic() {
List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
// mock1Setting = factory.createNewSetting("mock11", "mock1", new ArrayList<Dependency>(), new InterpreterOption(false), new Properties());
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
// get interpreter
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
// try to get unavailable interpreter
assertNull(interpreterSettingManager.get("unknown"));
// restart interpreter
interpreterSettingManager.restart(mock1Setting.getId());
assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session"));
}
@Test
public void testRemoteRepl() throws Exception {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
intp1Properties.put("PROPERTY_1",
new InterpreterProperty("PROPERTY_1", "VALUE_1"));
intp1Properties.put("property_2", new InterpreterProperty("property_2", "value_2"));
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
// get interpreter
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
assertTrue(interpreterGroup.get("session").get(0) instanceof LazyOpenInterpreter);
LazyOpenInterpreter lazyInterpreter = (LazyOpenInterpreter)(interpreterGroup.get("session").get(0));
assertTrue(lazyInterpreter.getInnerInterpreter() instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter = (RemoteInterpreter) lazyInterpreter.getInnerInterpreter();
assertEquals("VALUE_1", remoteInterpreter.getEnv().get("PROPERTY_1"));
assertEquals("value_2", remoteInterpreter.getProperty("property_2"));
}
/**
* 2 users' interpreters in scoped mode. Each user has one session. Restarting user1's interpreter
* won't affect user2's interpreter
* @throws Exception
*/
@Test
public void testRestartInterpreterInScopedMode() throws Exception {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
intp1Properties.put("PROPERTY_1",
new InterpreterProperty("PROPERTY_1", "VALUE_1"));
intp1Properties.put("property_2",
new InterpreterProperty("property_2", "value_2"));
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
mock1Setting.getOption().setPerUser("scoped");
mock1Setting.getOption().setPerNote("shared");
// set remote as false so that we won't create new remote interpreter process
mock1Setting.getOption().setRemote(false);
mock1Setting.getOption().setHost("localhost");
mock1Setting.getOption().setPort(2222);
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user1", "sharedProcess");
factory.createInterpretersForNote(mock1Setting, "user1", "sharedProcess", "user1");
factory.createInterpretersForNote(mock1Setting, "user2", "sharedProcess", "user2");
LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter)interpreterGroup.get("user1").get(0);
interpreter1.open();
LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup.get("user2").get(0);
interpreter2.open();
mock1Setting.closeAndRemoveInterpreterGroup("sharedProcess", "user1");
assertFalse(interpreter1.isOpen());
assertTrue(interpreter2.isOpen());
}
/**
* 2 users' interpreters in isolated mode. Each user has one interpreterGroup. Restarting user1's interpreter
* won't affect user2's interpreter
* @throws Exception
*/
@Test
public void testRestartInterpreterInIsolatedMode() throws Exception {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
intp1Properties.put("PROPERTY_1",
new InterpreterProperty("PROPERTY_1", "VALUE_1"));
intp1Properties.put("property_2",
new InterpreterProperty("property_2", "value_2"));
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
mock1Setting.getOption().setPerUser("isolated");
mock1Setting.getOption().setPerNote("shared");
// set remote as false so that we won't create new remote interpreter process
mock1Setting.getOption().setRemote(false);
mock1Setting.getOption().setHost("localhost");
mock1Setting.getOption().setPort(2222);
InterpreterGroup interpreterGroup1 = mock1Setting.getInterpreterGroup("user1", "note1");
InterpreterGroup interpreterGroup2 = mock1Setting.getInterpreterGroup("user2", "note2");
factory.createInterpretersForNote(mock1Setting, "user1", "note1", "shared_session");
factory.createInterpretersForNote(mock1Setting, "user2", "note2", "shared_session");
LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter)interpreterGroup1.get("shared_session").get(0);
interpreter1.open();
LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup2.get("shared_session").get(0);
interpreter2.open();
mock1Setting.closeAndRemoveInterpreterGroup("note1", "user1");
assertFalse(interpreter1.isOpen());
assertTrue(interpreter2.isOpen());
}
@Test
public void testFactoryDefaultList() throws IOException, RepositoryException {
// get default settings
List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
assertTrue(interpreterSettingManager.get().size() >= all.size());
}
@Test
public void testExceptions() throws InterpreterException, IOException, RepositoryException {
List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
// add setting with null option & properties expected nullArgumentException.class
try {
interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
} catch(NullArgumentException e) {
assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
}
try {
interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
} catch (NullArgumentException e){
assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage());
}
}
@Test
public void testSaveLoad() throws IOException, RepositoryException {
// interpreter settings
int numInterpreters = interpreterSettingManager.get().size();
// check if file saved
assertTrue(new File(conf.getInterpreterSettingPath()).exists());
interpreterSettingManager.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new HashMap<String, InterpreterProperty>());
assertEquals(numInterpreters + 1, interpreterSettingManager.get().size());
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
/*
Current situation, if InterpreterSettinfRef doesn't have the key of InterpreterSetting, it would be ignored.
Thus even though interpreter.json have several interpreterSetting in that file, it would be ignored and would not be initialized from loadFromFile.
In this case, only "mock11" would be referenced from file under interpreter/mock, and "mock11" group would be initialized.
*/
// TODO(jl): Decide how to handle the know referenced interpreterSetting.
assertEquals(1, interpreterSettingManager.get().size());
}
@Test
public void testInterpreterSettingPropertyClass() throws IOException, RepositoryException {
// check if default interpreter reference's property type is map
Map<String, InterpreterSetting> interpreterSettingRefs = interpreterSettingManager.getAvailableInterpreterSettings();
InterpreterSetting intpSetting = interpreterSettingRefs.get("mock1");
Map<String, DefaultInterpreterProperty> intpProperties =
(Map<String, DefaultInterpreterProperty>) intpSetting.getProperties();
assertTrue(intpProperties instanceof Map);
// check if interpreter instance is saved as Properties in conf/interpreter.json file
Map<String, InterpreterProperty> properties = new HashMap<String, InterpreterProperty>();
properties.put("key1", new InterpreterProperty("key1", "value1", "type1"));
properties.put("key2", new InterpreterProperty("key2", "value2", "type2"));
interpreterSettingManager.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties);
String confFilePath = conf.getInterpreterSettingPath();
byte[] encoded = Files.readAllBytes(Paths.get(confFilePath));
String json = new String(encoded, "UTF-8");
InterpreterInfoSaving infoSaving = InterpreterInfoSaving.fromJson(json);
Map<String, InterpreterSetting> interpreterSettings = infoSaving.interpreterSettings;
for (String key : interpreterSettings.keySet()) {
InterpreterSetting setting = interpreterSettings.get(key);
if (setting.getName().equals("newMock")) {
assertEquals(setting.getProperties().toString(), properties.toString());
}
}
}
@Test
public void testInterpreterAliases() throws IOException, RepositoryException {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null);
interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>() {{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
interpreterSettingManager.add("group2", new ArrayList<InterpreterInfo>(){{
add(info2);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2", null);
final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
final InterpreterSetting setting2 = interpreterSettingManager.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
interpreterSettingManager.setInterpreters("user", "note", new ArrayList<String>() {{
add(setting1.getId());
add(setting2.getId());
}});
assertEquals("className1", factory.getInterpreter("user1", "note", "test-group1").getClassName());
assertEquals("className1", factory.getInterpreter("user1", "note", "group1").getClassName());
}
@Test
public void testMultiUser() throws IOException, RepositoryException {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>(){{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED);
final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new HashMap<String, InterpreterProperty>());
interpreterSettingManager.setInterpreters("user1", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
interpreterSettingManager.setInterpreters("user2", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
assertNotEquals(factory.getInterpreter("user1", "note", "test-group1"), factory.getInterpreter("user2", "note", "test-group1"));
}
@Test
public void testInvalidInterpreterSettingName() {
try {
interpreterSettingManager.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new HashMap<String, InterpreterProperty>());
fail("expect fail because of invalid InterpreterSetting Name");
} catch (IOException e) {
assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
}
}
@Test
public void getEditorSetting() throws IOException, RepositoryException, SchedulerException {
List<String> intpIds = new ArrayList<>();
for(InterpreterSetting intpSetting: interpreterSettingManager.get()) {
if (intpSetting.getName().startsWith("mock1")) {
intpIds.add(intpSetting.getId());
}
}
Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous"));
Interpreter interpreter = factory.getInterpreter("user1", note.getId(), "mock11");
// get editor setting from interpreter-setting.json
Map<String, Object> editor = interpreterSettingManager.getEditorSetting(interpreter, "user1", note.getId(), "mock11");
assertEquals("java", editor.get("language"));
// when interpreter is not loaded via interpreter-setting.json
// or editor setting doesn't exit
editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock1"),"user1", note.getId(), "mock1");
assertEquals(null, editor.get("language"));
// when interpreter is not bound to note
editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock11"),"user1", note.getId(), "mock2");
assertEquals("text", editor.get("language"));
}
@Test
public void registerCustomInterpreterRunner() throws IOException {
InterpreterSettingManager spyInterpreterSettingManager = spy(interpreterSettingManager);
doNothing().when(spyInterpreterSettingManager).saveToFile();
ArrayList<InterpreterInfo> interpreterInfos1 = new ArrayList<>();
interpreterInfos1.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
spyInterpreterSettingManager.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, DefaultInterpreterProperty>newHashMap(), "/normalGroup1", null);
spyInterpreterSettingManager.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
when(mockInterpreterRunner.getPath()).thenReturn("custom-linux-path.sh");
spyInterpreterSettingManager.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, DefaultInterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner);
spyInterpreterSettingManager.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
spyInterpreterSettingManager.setInterpreters("anonymous", "noteCustome", spyInterpreterSettingManager.getDefaultInterpreterSettingList());
factory.getInterpreter("anonymous", "noteCustome", "customGroup1");
verify(mockInterpreterRunner, times(1)).getPath();
}
@Test
public void interpreterRunnerTest() {
InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
String testInterpreterRunner = "relativePath.sh";
when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner); // This test only for Linux
Interpreter i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
String interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
assertNotEquals(interpreterRunner, testInterpreterRunner);
testInterpreterRunner = "/AbsolutePath.sh";
when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner);
i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
assertEquals(interpreterRunner, testInterpreterRunner);
public void testUnknownRepl2() throws IOException {
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
assertNull(interpreterFactory.getInterpreter("user1", "note1", "unknown_repl"));
}
}

269
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java Normal file
View file

@ -0,0 +1,269 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.junit.Test;
import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.repository.RemoteRepository;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
public class InterpreterSettingManagerTest extends AbstractInterpreterTest {
@Test
public void testInitInterpreterSettingManager() throws IOException, RepositoryException {
assertEquals(5, interpreterSettingManager.get().size());
InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test");
assertEquals("test", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
assertEquals(2, interpreterSetting.getInterpreterInfos().size());
// 3 other builtin properties:
// * zeppelin.interpeter.output.limit
// * zeppelin.interpreter.localRepo
// * zeppelin.interpreter.max.poolsize
assertEquals(6, interpreterSetting.getJavaProperties().size());
assertEquals("value_1", interpreterSetting.getJavaProperties().getProperty("property_1"));
assertEquals("new_value_2", interpreterSetting.getJavaProperties().getProperty("property_2"));
assertEquals("value_3", interpreterSetting.getJavaProperties().getProperty("property_3"));
assertEquals("shared", interpreterSetting.getOption().perNote);
assertEquals("shared", interpreterSetting.getOption().perUser);
assertEquals(0, interpreterSetting.getDependencies().size());
assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
assertNotNull(interpreterSetting.getAppEventListener());
assertNotNull(interpreterSetting.getDependencyResolver());
assertNotNull(interpreterSetting.getInterpreterSettingManager());
List<RemoteRepository> repositories = interpreterSettingManager.getRepositories();
assertEquals(2, repositories.size());
assertEquals("central", repositories.get(0).getId());
// Load it again
InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf,
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
assertEquals(5, interpreterSettingManager2.get().size());
interpreterSetting = interpreterSettingManager2.getByName("test");
assertEquals("test", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
assertEquals(2, interpreterSetting.getInterpreterInfos().size());
assertEquals(6, interpreterSetting.getJavaProperties().size());
assertEquals("value_1", interpreterSetting.getJavaProperties().getProperty("property_1"));
assertEquals("new_value_2", interpreterSetting.getJavaProperties().getProperty("property_2"));
assertEquals("value_3", interpreterSetting.getJavaProperties().getProperty("property_3"));
assertEquals("shared", interpreterSetting.getOption().perNote);
assertEquals("shared", interpreterSetting.getOption().perUser);
assertEquals(0, interpreterSetting.getDependencies().size());
repositories = interpreterSettingManager2.getRepositories();
assertEquals(2, repositories.size());
assertEquals("central", repositories.get(0).getId());
}
@Test
public void testCreateUpdateRemoveSetting() throws IOException {
// create new interpreter setting
InterpreterOption option = new InterpreterOption();
option.setPerNote("scoped");
option.setPerUser("scoped");
Map<String, InterpreterProperty> properties = new HashMap<>();
properties.put("property_4", new InterpreterProperty("property_4","value_4"));
try {
interpreterSettingManager.createNewSetting("test2", "test", new ArrayList<Dependency>(), option, properties);
fail("Should fail due to interpreter already existed");
} catch (IOException e) {
assertTrue(e.getMessage().contains("already existed"));
}
interpreterSettingManager.createNewSetting("test3", "test", new ArrayList<Dependency>(), option, properties);
assertEquals(6, interpreterSettingManager.get().size());
InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test3");
assertEquals("test3", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
// 3 other builtin properties:
// * zeppelin.interpeter.output.limit
// * zeppelin.interpreter.localRepo
// * zeppelin.interpreter.max.poolsize
assertEquals(4, interpreterSetting.getJavaProperties().size());
assertEquals("value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
assertEquals("scoped", interpreterSetting.getOption().perNote);
assertEquals("scoped", interpreterSetting.getOption().perUser);
assertEquals(0, interpreterSetting.getDependencies().size());
assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
assertNotNull(interpreterSetting.getAppEventListener());
assertNotNull(interpreterSetting.getDependencyResolver());
assertNotNull(interpreterSetting.getInterpreterSettingManager());
// load it again, it should be saved in interpreter-setting.json. So we can restore it properly
InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf,
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
assertEquals(6, interpreterSettingManager2.get().size());
interpreterSetting = interpreterSettingManager2.getByName("test3");
assertEquals("test3", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
assertEquals(6, interpreterSetting.getJavaProperties().size());
assertEquals("value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
assertEquals("scoped", interpreterSetting.getOption().perNote);
assertEquals("scoped", interpreterSetting.getOption().perUser);
assertEquals(0, interpreterSetting.getDependencies().size());
// update interpreter setting
InterpreterOption newOption = new InterpreterOption();
newOption.setPerNote("scoped");
newOption.setPerUser("isolated");
Map<String, InterpreterProperty> newProperties = new HashMap<>(properties);
newProperties.put("property_4", new InterpreterProperty("property_4", "new_value_4"));
List<Dependency> newDependencies = new ArrayList<>();
newDependencies.add(new Dependency("com.databricks:spark-avro_2.11:3.1.0"));
interpreterSettingManager.setPropertyAndRestart(interpreterSetting.getId(), newOption, newProperties, newDependencies);
interpreterSetting = interpreterSettingManager.get(interpreterSetting.getId());
assertEquals("test3", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
assertEquals(4, interpreterSetting.getJavaProperties().size());
assertEquals("new_value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
assertEquals("scoped", interpreterSetting.getOption().perNote);
assertEquals("isolated", interpreterSetting.getOption().perUser);
assertEquals(1, interpreterSetting.getDependencies().size());
assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
assertNotNull(interpreterSetting.getAppEventListener());
assertNotNull(interpreterSetting.getDependencyResolver());
assertNotNull(interpreterSetting.getInterpreterSettingManager());
// restart in note page
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
interpreterSettingManager.setInterpreterBinding("user2", "note2", interpreterSettingManager.getSettingIds());
interpreterSettingManager.setInterpreterBinding("user3", "note3", interpreterSettingManager.getSettingIds());
// create 3 sessions as it is scoped mode
interpreterSetting.getOption().setPerUser("scoped");
interpreterSetting.getDefaultInterpreter("user1", "note1");
interpreterSetting.getDefaultInterpreter("user2", "note2");
interpreterSetting.getDefaultInterpreter("user3", "note3");
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
assertEquals(3, interpreterGroup.getSessionNum());
// only close user1's session
interpreterSettingManager.restart(interpreterSetting.getId(), "note1", "user1");
assertEquals(2, interpreterGroup.getSessionNum());
// close all the sessions
interpreterSettingManager.restart(interpreterSetting.getId(), "note1", "anonymous");
assertEquals(0, interpreterGroup.getSessionNum());
// remove interpreter setting
interpreterSettingManager.remove(interpreterSetting.getId());
assertEquals(5, interpreterSettingManager.get().size());
// load it again
InterpreterSettingManager interpreterSettingManager3 = new InterpreterSettingManager(new ZeppelinConfiguration(),
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
assertEquals(5, interpreterSettingManager3.get().size());
}
@Test
public void testInterpreterBinding() throws IOException {
assertNull(interpreterSettingManager.getInterpreterBinding("note1"));
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
assertEquals(interpreterSettingManager.getInterpreterSettingIds(), interpreterSettingManager.getInterpreterBinding("note1"));
}
@Test
public void testUpdateInterpreterBinding_PerNoteShared() throws IOException {
InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
defaultInterpreterSetting.getOption().setPerNote("shared");
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
// create interpreter of the first binded interpreter setting
interpreterFactory.getInterpreter("user1", "note1", "");
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
// choose the first setting
List<String> newSettingIds = new ArrayList<>();
newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
// InterpreterGroup will still be alive as it is shared
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testUpdateInterpreterBinding_PerNoteIsolated() throws IOException {
InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
defaultInterpreterSetting.getOption().setPerNote("isolated");
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
// create interpreter of the first binded interpreter setting
interpreterFactory.getInterpreter("user1", "note1", "");
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
// choose the first setting
List<String> newSettingIds = new ArrayList<>();
newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
// InterpreterGroup will be closed as it is only belong to this note
assertEquals(0, defaultInterpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testUpdateInterpreterBinding_PerNoteScoped() throws IOException {
InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
defaultInterpreterSetting.getOption().setPerNote("scoped");
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
interpreterSettingManager.setInterpreterBinding("user1", "note2", interpreterSettingManager.getInterpreterSettingIds());
// create 2 interpreter of the first binded interpreter setting for note1 and note2
interpreterFactory.getInterpreter("user1", "note1", "");
interpreterFactory.getInterpreter("user1", "note2", "");
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, defaultInterpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// choose the first setting
List<String> newSettingIds = new ArrayList<>();
newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
// InterpreterGroup will be still alive but session belong to note1 will be closed
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
}
}

570
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
View file

@ -1,327 +1,411 @@
package org.apache.zeppelin.interpreter;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
package org.apache.zeppelin.interpreter;
import org.junit.Test;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.junit.Assert.assertNull;
public class InterpreterSettingTest {
@Test
public void sharedModeCloseandRemoveInterpreterGroupTest() {
public void testCreateInterpreters() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SHARED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterOption.setRemote(false);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
// create default interpreter for user1 and note1
assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getDefaultInterpreter("user1", "note1").getClassName());
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
// create interpreter echo for user1 and note1
assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "echo").getClassName());
assertEquals(interpreterSetting.getDefaultInterpreter("user1", "note1"), interpreterSetting.getInterpreter("user1", "note1", "echo"));
// This won't effect anything
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
// create interpreter double_echo for user1 and note1
assertEquals(DoubleEchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "double_echo").getClassName());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
// create non-existed interpreter
assertNull(interpreterSetting.getInterpreter("user1", "note1", "invalid_echo"));
}
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
@Test
public void testSharedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SHARED);
interpreterOption.setRemote(false);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create default interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// create default interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// create default interpreter user1 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// only 1 session is created, this session is shared across users and notes
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeInterpreters("note1", "user1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perUserScopedModeCloseAndRemoveInterpreterGroupTest() {
public void testPerUserScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(2, interpreterSetting.getInterpreterGroup("user2", "note1").size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// Check if non-existed key works or not
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
interpreterSetting.closeInterpreters("user1", "note1");
// InterpreterGroup is still there, but one session is removed
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perUserIsolatedModeCloseAndRemoveInterpreterGroupTest() {
public void testPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.SCOPED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user1 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeInterpreters("user1", "note1");
// InterpreterGroup is still there, but one session is removed
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testPerUserIsolatedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
// Each user own one InterpreterGroup and one session per InterpreterGroup
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(1).getSessionNum());
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perNoteScopedModeCloseAndRemoveInterpreterGroupTest() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.SCOPED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note2");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note2"), interpreterList2);
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note2").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
// Check if non-existed key works or not
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note2", "user1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perNoteIsolatedModeCloseAndRemoveInterpreterGroupTest() {
public void testPerNoteIsolatedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.ISOLATED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note2");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note2"), interpreterList2);
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note2").size());
// Each user own one InterpreterGroup and one session per InterpreterGroup
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(1).getSessionNum());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
interpreterSetting.closeAndRemoveInterpreterGroup("note2", "user1");
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perNoteScopedModeRemoveInterpreterGroupWhenNoteIsRemoved() {
public void testPerUserIsolatedPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
interpreterOption.setPerNote(InterpreterOption.SCOPED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// This method will be called when remove note
interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
// group1 for user1 has 2 sessions, and group2 for user2 has 1 session
assertEquals(interpreterSetting.getInterpreterGroup("user1", "note1"), interpreterSetting.getInterpreterGroup("user1", "note2"));
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").getSessionNum());
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note2").getSessionNum());
assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").getSessionNum());
// close one session for user1
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").getSessionNum());
// close another session for user1
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// close session for user2
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
// Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
assertEquals(0, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
@Test
public void perNoteIsolatedModeRemoveInterpreterGroupWhenNoteIsRemoved() {
public void testPerUserIsolatedPerNoteIsolatedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
interpreterOption.setPerNote(InterpreterOption.ISOLATED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
// This method will be called when remove note
interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
// create interpreter for user1 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(3, interpreterSetting.getAllInterpreterGroups().size());
// create interpreter for user2 and note2
interpreterSetting.getDefaultInterpreter("user2", "note2");
assertEquals(4, interpreterSetting.getAllInterpreterGroups().size());
for (InterpreterGroup interpreterGroup : interpreterSetting.getAllInterpreterGroups()) {
// each InterpreterGroup has one session
assertEquals(1, interpreterGroup.getSessionNum());
}
// close one session for user1 and note1
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(3, interpreterSetting.getAllInterpreterGroups().size());
// close one session for user1 and note2
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
// close one session for user2 and note1
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// close one session for user2 and note2
interpreterSetting.closeInterpreters("user2", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
// Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
assertEquals(0, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
@Test
public void perUserScopedModeNeverRemoveInterpreterGroupWhenNoteIsRemoved() {
public void testPerUserScopedPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
interpreterOption.setPerNote(InterpreterOption.SCOPED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// This method will be called when remove note
interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
// create interpreter for user1 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
@Test
public void perUserIsolatedModeNeverRemoveInterpreterGroupWhenNoteIsRemoved() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(3, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// This method will be called when remove note
interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
// create interpreter for user2 and note2
interpreterSetting.getDefaultInterpreter("user2", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note1").size());
assertEquals(4, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// close one session for user1 and note1
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(3, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// close one session for user1 and note2
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// close one session for user2 and note1
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// close one session for user2 and note2
interpreterSetting.closeInterpreters("user2", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
}

90
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroupTest.java Normal file
View file

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static org.junit.Assert.assertEquals;
public class ManagedInterpreterGroupTest {
private static final Logger LOGGER = LoggerFactory.getLogger(ManagedInterpreterGroupTest.class);
private InterpreterSetting interpreterSetting;
@Before
public void setUp() throws IOException, RepositoryException {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
interpreterOption.setRemote(false);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
}
@Test
public void testInterpreterGroup() {
ManagedInterpreterGroup interpreterGroup = new ManagedInterpreterGroup("group_1", interpreterSetting);
assertEquals(0, interpreterGroup.getSessionNum());
// create session_1
List<Interpreter> interpreters = interpreterGroup.getOrCreateSession("user1", "session_1");
assertEquals(2, interpreters.size());
assertEquals(EchoInterpreter.class.getName(), interpreters.get(0).getClassName());
assertEquals(DoubleEchoInterpreter.class.getName(), interpreters.get(1).getClassName());
assertEquals(1, interpreterGroup.getSessionNum());
// get the same interpreters when interpreterGroup.getOrCreateSession is invoked again
assertEquals(interpreters, interpreterGroup.getOrCreateSession("user1", "session_1"));
assertEquals(1, interpreterGroup.getSessionNum());
// create session_2
List<Interpreter> interpreters2 = interpreterGroup.getOrCreateSession("user1", "session_2");
assertEquals(2, interpreters2.size());
assertEquals(EchoInterpreter.class.getName(), interpreters2.get(0).getClassName());
assertEquals(DoubleEchoInterpreter.class.getName(), interpreters2.get(1).getClassName());
assertEquals(2, interpreterGroup.getSessionNum());
// close session_1
interpreterGroup.close("session_1");
assertEquals(1, interpreterGroup.getSessionNum());
// close InterpreterGroup
interpreterGroup.close();
assertEquals(0, interpreterGroup.getSessionNum());
}
}

60
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java Normal file
View file

@ -0,0 +1,60 @@
package org.apache.zeppelin.interpreter;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import java.util.Properties;
/**
* Interpreter that only accept long value and sleep for such period
*/
public class SleepInterpreter extends Interpreter {
public SleepInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
try {
Thread.sleep(Long.parseLong(st));
return new InterpreterResult(InterpreterResult.Code.SUCCESS);
} catch (Exception e) {
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public Scheduler getScheduler() {
if (Boolean.parseBoolean(property.getProperty("zeppelin.SleepInterpreter.parallel", "false"))) {
return SchedulerFactory.singleton().createOrGetParallelScheduler(
"Parallel-" + SleepInterpreter.class.getName(), 10);
}
return super.getScheduler();
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}

15
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
View file

@ -17,11 +17,6 @@
package org.apache.zeppelin.interpreter.mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -29,8 +24,14 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
public class MockInterpreter1 extends Interpreter{
Map<String, Object> vars = new HashMap<>();
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
public class MockInterpreter1 extends Interpreter {
Map<String, Object> vars = new HashMap<>();
public MockInterpreter1(Properties property) {
super(property);

20
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
View file

@ -17,10 +17,6 @@
package org.apache.zeppelin.interpreter.mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
@ -29,12 +25,18 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
public class MockInterpreter11 extends Interpreter{
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
public class MockInterpreter11 extends Interpreter {
Map<String, Object> vars = new HashMap<>();
public MockInterpreter11(Properties property) {
super(property);
}
boolean open;
@Override
@ -53,7 +55,7 @@ public class MockInterpreter11 extends Interpreter{
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl11: "+st);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl11: " + st);
}
@Override
@ -72,12 +74,12 @@ public class MockInterpreter11 extends Interpreter{
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_" + this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
InterpreterContext interpreterContext) {
return null;
}
}
}

10
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
View file

@ -17,11 +17,6 @@
package org.apache.zeppelin.interpreter.mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -29,6 +24,11 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
public class MockInterpreter2 extends Interpreter{
Map<String, Object> vars = new HashMap<>();

30
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
View file

@ -17,23 +17,6 @@
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@ -43,6 +26,19 @@ import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.*;
public class AppendOutputRunnerTest {
private static final int NUM_EVENTS = 10000;

87
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
View file

@ -17,15 +17,10 @@
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.zeppelin.display.*;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular;
import org.apache.zeppelin.resource.LocalResourcePool;
@ -34,17 +29,25 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup;
private HashMap<String, String> env;
private RemoteInterpreter intp;
private InterpreterContext context;
private RemoteAngularObjectRegistry localRegistry;
private InterpreterSetting interpreterSetting;
private AtomicInteger onAdd;
private AtomicInteger onUpdate;
@ -56,32 +59,24 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
onUpdate = new AtomicInteger(0);
onRemove = new AtomicInteger(0);
intpGroup = new InterpreterGroup("intpId");
localRegistry = new RemoteAngularObjectRegistry("intpId", this, intpGroup);
intpGroup.setAngularObjectRegistry(localRegistry);
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterAngular.class.getName(), "mock", true, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
interpreterSetting = new InterpreterSetting.Builder()
.setId("test")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.setRunner(runner)
.setInterpreterDir("../interpeters/test")
.create();
Properties p = new Properties();
intp = new RemoteInterpreter(
p,
"note",
MockInterpreterAngular.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false
);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intp);
intp.setInterpreterGroup(intpGroup);
intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
localRegistry = (RemoteAngularObjectRegistry) intp.getInterpreterGroup().getAngularObjectRegistry();
context = new InterpreterContext(
"note",
@ -92,17 +87,17 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new AngularObjectRegistry(intp.getInterpreterGroup().getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
intp.open();
}
@After
public void tearDown() throws Exception {
intp.close();
intpGroup.close();
interpreterSetting.close();
}
@Test
@ -147,7 +142,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
Thread.sleep(500); // waitFor eventpoller pool event
String[] result = ret.message().get(0).getData().split(" ");
assertEquals("0", result[0]); // size of registry
// create object
ret = intp.interpret("add n1 v1", context);
Thread.sleep(500);
@ -172,11 +167,11 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
Thread.sleep(500); // waitFor eventpoller pool event
String[] result = ret.message().get(0).getData().split(" ");
assertEquals("0", result[0]); // size of registry
// create object
localRegistry.addAndNotifyRemoteProcess("n1", "v1", "note", null);
// get from remote registry
// get from remote registry
ret = intp.interpret("get", context);
Thread.sleep(500); // waitFor eventpoller pool event
result = ret.message().get(0).getData().split(" ");

65
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
View file

@ -17,22 +17,18 @@
package org.apache.zeppelin.interpreter.remote;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterOutputStream;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Properties;
import java.util.*;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
/**
@ -43,41 +39,32 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup;
private HashMap<String, String> env;
private InterpreterSetting interpreterSetting;
@Before
public void setUp() throws Exception {
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
InterpreterOption interpreterOption = new InterpreterOption();
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterOutputStream.class.getName(), "mock", true, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
interpreterSetting = new InterpreterSetting.Builder()
.setId("test")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.setRunner(runner)
.setInterpreterDir("../interpeters/test")
.create();
}
@After
public void tearDown() throws Exception {
intpGroup.close();
}
private RemoteInterpreter createMockInterpreter() {
RemoteInterpreter intp = new RemoteInterpreter(
new Properties(),
"note",
MockInterpreterOutputStream.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
this,
null,
"anonymous",
false);
intpGroup.get("note").add(intp);
intp.setInterpreterGroup(intpGroup);
return intp;
interpreterSetting.close();
}
private InterpreterContext createInterpreterContext() {
@ -90,14 +77,14 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
null,
new LinkedList<InterpreterContextRunner>(), null);
}
@Test
public void testInterpreterResultOnly() {
RemoteInterpreter intp = createMockInterpreter();
RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS::staticresult", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("staticresult", ret.message().get(0).getData());
@ -113,7 +100,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testInterpreterOutputStreamOnly() {
RemoteInterpreter intp = createMockInterpreter();
RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:streamresult:", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("streamresult", ret.message().get(0).getData());
@ -125,7 +112,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testInterpreterResultOutputStreamMixed() {
RemoteInterpreter intp = createMockInterpreter();
RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:stream:static", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("stream", ret.message().get(0).getData());
@ -134,7 +121,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testOutputType() {
RemoteInterpreter intp = createMockInterpreter();
RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:%html hello:", createInterpreterContext());
assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());

131
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
View file

@ -1,131 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.*;
import java.util.HashMap;
import java.util.Properties;
import org.apache.thrift.TException;
import org.apache.thrift.transport.TTransportException;
import org.apache.zeppelin.interpreter.Constants;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.junit.Test;
public class RemoteInterpreterProcessTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private static final int DUMMY_PORT=3678;
@Test
public void testStartStop() {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
10 * 1000, null, null,"fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
assertEquals(2, rip.reference(intpGroup, "anonymous", false));
assertEquals(true, rip.isRunning());
assertEquals(1, rip.dereference());
assertEquals(true, rip.isRunning());
assertEquals(0, rip.dereference());
assertEquals(false, rip.isRunning());
}
@Test
public void testClientFactory() throws Exception {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class), 10 * 1000, "fakeName");
rip.reference(intpGroup, "anonymous", false);
assertEquals(0, rip.getNumActiveClient());
assertEquals(0, rip.getNumIdleClient());
Client client = rip.getClient();
assertEquals(1, rip.getNumActiveClient());
assertEquals(0, rip.getNumIdleClient());
rip.releaseClient(client);
assertEquals(0, rip.getNumActiveClient());
assertEquals(1, rip.getNumIdleClient());
rip.dereference();
}
@Test
public void testStartStopRemoteInterpreter() throws TException, InterruptedException {
RemoteInterpreterServer server = new RemoteInterpreterServer(3678);
server.start();
boolean running = false;
long startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < 10 * 1000) {
if (server.isRunning()) {
running = true;
break;
} else {
Thread.sleep(200);
}
}
Properties properties = new Properties();
properties.setProperty(Constants.ZEPPELIN_INTERPRETER_PORT, "3678");
properties.setProperty(Constants.ZEPPELIN_INTERPRETER_HOST, "localhost");
InterpreterGroup intpGroup = mock(InterpreterGroup.class);
when(intpGroup.getProperty()).thenReturn(properties);
when(intpGroup.containsKey(Constants.EXISTING_PROCESS)).thenReturn(true);
RemoteInterpreterProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT,
"nonexists",
"fakeRepo",
new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class)
, 10 * 1000,
"fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
assertEquals(true, rip.isRunning());
}
@Test
public void testPropagateError() throws TException, InterruptedException {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
"echo hello_world", "nonexists", "fakeRepo", new HashMap<String, String>(),
10 * 1000, null, null, "fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
try {
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
} catch (InterpreterException e) {
e.getMessage().contains("hello_world");
}
assertEquals(0, rip.referenceCount());
}
}

1114
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
View file

File diff suppressed because it is too large Load diff

36
zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java → zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetAngularObjectSizeInterpreter.java
View file

@ -14,62 +14,50 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
package org.apache.zeppelin.interpreter.remote.mock;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
public class MockInterpreter1 extends Interpreter{
Map<String, Object> vars = new HashMap<>();
import java.util.Properties;
public MockInterpreter1(Properties property) {
public class GetAngularObjectSizeInterpreter extends Interpreter {
public GetAngularObjectSizeInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: "+st);
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
"" + context.getAngularObjectRegistry().getRegistry().size());
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
return FormType.NATIVE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return null;
}
}

12
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java → zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java
View file

@ -16,7 +16,9 @@
*/
package org.apache.zeppelin.interpreter.remote.mock;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@ -25,9 +27,9 @@ import java.util.List;
import java.util.Properties;
public class MockInterpreterEnv extends Interpreter {
public class GetEnvPropertyInterpreter extends Interpreter {
public MockInterpreterEnv(Properties property) {
public GetEnvPropertyInterpreter(Properties property) {
super(property);
}
@ -43,9 +45,9 @@ public class MockInterpreterEnv extends Interpreter {
public InterpreterResult interpret(String st, InterpreterContext context) {
String[] cmd = st.split(" ");
if (cmd[0].equals("getEnv")) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]));
return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]) == null ? "null" : System.getenv(cmd[1]));
} else if (cmd[0].equals("getProperty")){
return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]));
return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]) == null ? "null" : System.getProperty(cmd[1]));
} else {
return new InterpreterResult(InterpreterResult.Code.ERROR, cmd[0]);
}

7
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
View file

@ -17,19 +17,18 @@
package org.apache.zeppelin.interpreter.remote.mock;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import java.util.List;
import java.util.Properties;
public class MockInterpreterA extends Interpreter {
private String lastSt;

9
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
View file

@ -17,19 +17,18 @@
package org.apache.zeppelin.interpreter.remote.mock;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectWatcher;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
public class MockInterpreterAngular extends Interpreter {
AtomicInteger numWatch = new AtomicInteger(0);

126
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
View file

@ -1,126 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote.mock;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
public class MockInterpreterB extends Interpreter {
public MockInterpreterB(Properties property) {
super(property);
}
@Override
public void open() {
//new RuntimeException().printStackTrace();
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
MockInterpreterA intpA = getInterpreterA();
String intpASt = intpA.getLastStatement();
long timeToSleep = Long.parseLong(st);
if (intpASt != null) {
timeToSleep += Long.parseLong(intpASt);
}
try {
Thread.sleep(timeToSleep);
} catch (NumberFormatException | InterruptedException e) {
throw new InterpreterException(e);
}
return new InterpreterResult(Code.SUCCESS, Long.toString(timeToSleep));
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return null;
}
public MockInterpreterA getInterpreterA() {
InterpreterGroup interpreterGroup = getInterpreterGroup();
synchronized (interpreterGroup) {
for (List<Interpreter> interpreters : interpreterGroup.values()) {
boolean belongsToSameNoteGroup = false;
MockInterpreterA a = null;
for (Interpreter intp : interpreters) {
if (intp.getClassName().equals(MockInterpreterA.class.getName())) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
a = (MockInterpreterA) p;
}
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
if (this == p) {
belongsToSameNoteGroup = true;
}
}
if (belongsToSameNoteGroup) {
return a;
}
}
}
return null;
}
@Override
public Scheduler getScheduler() {
MockInterpreterA intpA = getInterpreterA();
if (intpA != null) {
return intpA.getScheduler();
}
return null;
}
}

5
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
View file

@ -16,7 +16,10 @@
*/
package org.apache.zeppelin.interpreter.remote.mock;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;

11
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
View file

@ -17,22 +17,19 @@
package org.apache.zeppelin.interpreter.remote.mock;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.gson.Gson;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectWatcher;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePool;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
public class MockInterpreterResourcePool extends Interpreter {
AtomicInteger numWatch = new AtomicInteger(0);

243
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
View file

@ -1,243 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter11;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
public class NoteInterpreterLoaderTest {
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterFactory factory;
private InterpreterSettingManager interpreterSettingManager;
private DependencyResolver depResolver;
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
new File(tmpDir, "conf").mkdirs();
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
conf = ZeppelinConfiguration.create();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, Maps.<String, Object>newHashMap()));
interpreterInfos.add(new InterpreterInfo(MockInterpreter11.class.getName(), "mock11", false, Maps.<String, Object>newHashMap()));
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, Maps.<String, Object>newHashMap()));
interpreterSettingManager.add("group1", interpreterInfos, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock", null);
interpreterSettingManager.add("group2", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock", null);
interpreterSettingManager.createNewSetting("group1", "group1", Lists.<Dependency>newArrayList(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
interpreterSettingManager.createNewSetting("group2", "group2", Lists.<Dependency>newArrayList(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
Interpreter.registeredInterpreters.clear();
}
@Test
public void testGetInterpreter() throws IOException {
interpreterSettingManager.setInterpreters("user", "note", interpreterSettingManager.getDefaultInterpreterSettingList());
// when there're no interpreter selection directive
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", " ").getClassName());
// when group name is omitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "mock11").getClassName());
// when 'name' is ommitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2").getClassName());
// when nothing is ommitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1.mock1").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName());
interpreterSettingManager.closeNote("user", "note");
}
@Test
public void testNoteSession() throws IOException {
interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
// interpreters are not created before accessing it
assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
factory.getInterpreter("user", "noteA", null).open();
factory.getInterpreter("user", "noteB", null).open();
assertTrue(
factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
// interpreters are created after accessing it
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
// invalid close
interpreterSettingManager.closeNote("user", "note");
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
// when
interpreterSettingManager.closeNote("user", "noteA");
interpreterSettingManager.closeNote("user", "noteB");
// interpreters are destroyed after close
assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
}
@Test
public void testNotePerInterpreterProcess() throws IOException {
interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
// interpreters are not created before accessing it
assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
factory.getInterpreter("user", "noteA", null).open();
factory.getInterpreter("user", "noteB", null).open();
// per note interpreter process
assertFalse(
factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
// interpreters are created after accessing it
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
// when
interpreterSettingManager.closeNote("user", "noteA");
interpreterSettingManager.closeNote("user", "noteB");
// interpreters are destroyed after close
assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
}
@Test
public void testNoteInterpreterCloseForAll() throws IOException {
interpreterSettingManager.setInterpreters("user", "FitstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
interpreterSettingManager.setInterpreters("user", "yourFirstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
// interpreters are not created before accessing it
assertNull(interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getInterpreterGroup("user", "FitstNote").get("FitstNote"));
assertNull(interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getInterpreterGroup("user", "yourFirstNote").get("yourFirstNote"));
Interpreter firstNoteIntp = factory.getInterpreter("user", "FitstNote", "group1.mock1");
Interpreter yourFirstNoteIntp = factory.getInterpreter("user", "yourFirstNote", "group1.mock1");
firstNoteIntp.open();
yourFirstNoteIntp.open();
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
interpreterSettingManager.closeNote("user", "FitstNote");
assertFalse(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
//reopen
firstNoteIntp.open();
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
// invalid check
interpreterSettingManager.closeNote("invalid", "Note");
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
// invalid contains value check
interpreterSettingManager.closeNote("u", "Note");
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
}
private void delete(File file){
if(file.isFile()) file.delete();
else if(file.isDirectory()){
File [] files = file.listFiles();
if(files!=null && files.length>0){
for(File f : files){
delete(f);
}
}
file.delete();
}
}
}

172
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
View file

@ -17,31 +17,28 @@
package org.apache.zeppelin.notebook;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@ -56,18 +53,35 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
public class NotebookTest implements JobListenerFactory{
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
public class NotebookTest extends AbstractInterpreterTest implements JobListenerFactory {
private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class);
private File tmpDir;
private ZeppelinConfiguration conf;
private SchedulerFactory schedulerFactory;
private File notebookDir;
private Notebook notebook;
private NotebookRepo notebookRepo;
private InterpreterFactory factory;
private InterpreterSettingManager interpreterSettingManager;
private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS;
@ -75,57 +89,30 @@ public class NotebookTest implements JobListenerFactory{
@Before
public void setUp() throws Exception {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true");
System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
super.setUp();
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
new File(tmpDir, "conf").mkdirs();
notebookDir = new File(tmpDir + "/notebook");
notebookDir.mkdirs();
System.setProperty(ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.toString() + "/conf");
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
conf = ZeppelinConfiguration.create();
this.schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
schedulerFactory = SchedulerFactory.singleton();
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search,
notebookAuthorization, credentials);
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true");
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
super.tearDown();
}
@Test
public void testSelectingReplImplementation() throws IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// run with default repl
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -259,7 +246,7 @@ public class NotebookTest implements JobListenerFactory{
Notebook notebook2 = new Notebook(
conf, notebookRepo, schedulerFactory,
new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager),
new InterpreterFactory(interpreterSettingManager),
interpreterSettingManager, null, null, null, null);
assertEquals(1, notebook2.getAllNotes().size());
@ -316,7 +303,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testRunAll() throws IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// p1
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -355,7 +342,7 @@ public class NotebookTest implements JobListenerFactory{
public void testSchedule() throws InterruptedException, IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
@ -428,8 +415,8 @@ public class NotebookTest implements JobListenerFactory{
public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
p.setConfig(config);
@ -448,21 +435,17 @@ public class NotebookTest implements JobListenerFactory{
notebook.refreshCron(note.getId());
MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter)
((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter())
.getInnerInterpreter()));
RemoteInterpreter mock1 = (RemoteInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock1");
MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter)
((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter())
.getInnerInterpreter()));
RemoteInterpreter mock2 = (RemoteInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock2");
// wait until interpreters are started
while (!mock1.isOpen() || !mock2.isOpen()) {
while (!mock1.isOpened() || !mock2.isOpened()) {
Thread.yield();
}
// wait until interpreters are closed
while (mock1.isOpen() || mock2.isOpen()) {
while (mock1.isOpened() || mock2.isOpened()) {
Thread.yield();
}
@ -481,7 +464,7 @@ public class NotebookTest implements JobListenerFactory{
public void testExportAndImportNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
String simpleText = "hello world";
@ -520,7 +503,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@ -554,7 +537,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous);
assertEquals(cloneNote.getName(), "Note " + cloneNote.getId());
@ -566,7 +549,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@ -591,28 +574,28 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testResourceRemovealOnParagraphNoteRemove() throws IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
}
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello");
Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p2.setText("%mock2 world");
for (InterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
}
note.runAll();
while (p1.isTerminated() == false || p1.getResult() == null) Thread.yield();
while (p2.isTerminated() == false || p2.getResult() == null) Thread.yield();
assertEquals(2, ResourcePoolUtils.getAllResources().size());
assertEquals(2, interpreterSettingManager.getAllResources().size());
// remove a paragraph
note.removeParagraph(anonymous.getUser(), p1.getId());
assertEquals(1, ResourcePoolUtils.getAllResources().size());
assertEquals(1, interpreterSettingManager.getAllResources().size());
// remove note
notebook.removeNote(note.getId(), anonymous);
assertEquals(0, ResourcePoolUtils.getAllResources().size());
assertEquals(0, interpreterSettingManager.getAllResources().size());
}
@Test
@ -620,10 +603,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -653,10 +636,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -687,10 +670,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// add local scope object
@ -700,14 +683,13 @@ public class NotebookTest implements JobListenerFactory{
// restart interpreter
interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId());
registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0)
.getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// local and global scope object should be removed
// But InterpreterGroup does not implement angularObjectRegistry per session (scoped, isolated)
// So for now, does not have good way to remove all objects in particular session on restart.
assertNotNull(registry.get("o1", note.getId(), null));
assertNotNull(registry.get("o2", null, null));
// New InterpreterGroup will be created and its AngularObjectRegistry will be created
assertNull(registry.get("o1", note.getId(), null));
assertNull(registry.get("o2", null, null));
notebook.removeNote(note.getId(), anonymous);
}
@ -815,7 +797,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException,
IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// create three paragraphs
Paragraph p1 = note.addNewParagraph(anonymous);

10
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
View file

@ -21,6 +21,7 @@ package org.apache.zeppelin.notebook;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
@ -39,7 +40,7 @@ import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectBuilder;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter.FormType;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterGroup;
@ -47,10 +48,7 @@ import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSetting.Status;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
@ -173,7 +171,7 @@ public class ParagraphTest {
Interpreter mockInterpreter = mock(Interpreter.class);
doReturn(mockInterpreter).when(spyParagraph).getRepl(anyString());
InterpreterGroup mockInterpreterGroup = mock(InterpreterGroup.class);
ManagedInterpreterGroup mockInterpreterGroup = mock(ManagedInterpreterGroup.class);
when(mockInterpreter.getInterpreterGroup()).thenReturn(mockInterpreterGroup);
when(mockInterpreterGroup.getId()).thenReturn("mock_id_1");
when(mockInterpreterGroup.getAngularObjectRegistry()).thenReturn(mock(AngularObjectRegistry.class));
@ -186,7 +184,7 @@ public class ParagraphTest {
when(mockInterpreterOption.permissionIsSet()).thenReturn(false);
when(mockInterpreterSetting.getStatus()).thenReturn(Status.READY);
when(mockInterpreterSetting.getId()).thenReturn("mock_id_1");
when(mockInterpreterSetting.getInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
when(mockInterpreterSetting.getOrCreateInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
spyInterpreterSettingList.add(mockInterpreterSetting);
when(mockNote.getId()).thenReturn("any_id");
when(mockInterpreterSettingManager.getInterpreterSettings(anyString())).thenReturn(spyInterpreterSettingList);

75
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
View file

@ -33,10 +33,12 @@ import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
@ -69,7 +71,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
private Credentials credentials;
private AuthenticationInfo anonymous;
private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
@Before
public void setUp() throws Exception {
String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis();
@ -91,12 +93,13 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
LOG.info("secondary note dir : " + secNotePath);
conf = ZeppelinConfiguration.create();
this.schedulerFactory = new SchedulerFactory();
this.schedulerFactory = SchedulerFactory.singleton();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
interpreterSettingManager = new InterpreterSettingManager(conf,
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
factory = new InterpreterFactory(interpreterSettingManager);
search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
@ -110,19 +113,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void tearDown() throws Exception {
delete(mainZepDir);
}
@Test
public void testRepoCount() throws IOException {
assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount());
}
@Test
public void testSyncOnCreate() throws IOException {
/* check that both storage systems are empty */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
/* create note */
Note note = notebookSync.createNote(anonymous);
@ -130,7 +133,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous);
}
@ -140,26 +143,26 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
Note note = notebookSync.createNote(anonymous);
/* check that created in both storage systems */
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
/* remove Note */
notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous);
/* check that deleted in both storages */
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
}
@Test
public void testSyncUpdateMain() throws IOException {
/* create note */
Note note = notebookSync.createNote(anonymous);
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -167,19 +170,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
/* new paragraph exists in note instance */
assertEquals(1, note.getParagraphs().size());
/* new paragraph not yet saved into storages */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
/* save to storage under index 0 (first storage) */
/* save to storage under index 0 (first storage) */
notebookRepoSync.save(0, note, anonymous);
/* check paragraph saved to first storage */
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
@ -284,45 +287,45 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
// one git versioned storage initialized
assertThat(vRepoSync.getRepoCount()).isEqualTo(1);
assertThat(vRepoSync.getRepo(0)).isInstanceOf(GitNotebookRepo.class);
GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0);
// no notes
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(0);
// create note
Note note = vNotebookSync.createNote(anonymous);
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(1);
String noteId = vRepoSync.list(anonymous).get(0).getId();
// first checkpoint
vRepoSync.checkpoint(noteId, "checkpoint message", anonymous);
int vCount = gitRepo.revisionHistory(noteId, anonymous).size();
assertThat(vCount).isEqualTo(1);
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
p.setConfig(config);
p.setText("%md checkpoint test");
// save and checkpoint again
vRepoSync.save(note, anonymous);
vRepoSync.checkpoint(noteId, "checkpoint message 2", anonymous);
assertThat(gitRepo.revisionHistory(noteId, anonymous).size()).isEqualTo(vCount + 1);
notebookRepoSync.remove(note.getId(), anonymous);
}
@Test
public void testSyncWithAcl() throws IOException {
/* scenario 1 - note exists with acl on main storage */
AuthenticationInfo user1 = new AuthenticationInfo("user1");
Note note = notebookSync.createNote(user1);
assertEquals(0, note.getParagraphs().size());
// saved on both storages
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
/* check that user1 is the only owner */
NotebookAuthorization authInfo = NotebookAuthorization.getInstance();
Set<String> entity = new HashSet<String>();
@ -332,23 +335,23 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getRunners(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
/* update note and save on secondary storage */
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello world");
assertEquals(1, note.getParagraphs().size());
notebookRepoSync.save(1, note, null);
/* check paragraph isn't saved into first storage */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
/* check paragraph is saved into second storage */
assertEquals(1, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
/* now sync by user1 */
notebookRepoSync.sync(user1);
/* check that note updated and acl are same on main storage*/
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
@ -357,7 +360,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getRunners(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
/* scenario 2 - note doesn't exist on main storage */
/* remove from main storage */
notebookRepoSync.remove(0, note.getId(), user1);
@ -368,7 +371,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getRunners(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
/* now sync - should bring note from secondary storage with added acl */
notebookRepoSync.sync(user1);
assertEquals(1, notebookRepoSync.list(0, null).size());
@ -428,5 +431,5 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
}
};
}
}

64
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
View file

@ -22,25 +22,14 @@ import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
@ -58,59 +47,34 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap;
public class VFSNotebookRepoTest implements JobListenerFactory {
public class VFSNotebookRepoTest extends AbstractInterpreterTest implements JobListenerFactory {
private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepoTest.class);
private ZeppelinConfiguration conf;
private SchedulerFactory schedulerFactory;
private Notebook notebook;
private NotebookRepo notebookRepo;
private InterpreterSettingManager interpreterSettingManager;
private InterpreterFactory factory;
private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
private File mainZepDir;
private File mainNotebookDir;
@Before
public void setUp() throws Exception {
String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis();
mainZepDir = new File(zpath);
mainZepDir.mkdirs();
new File(mainZepDir, "conf").mkdirs();
String mainNotePath = zpath + "/notebook";
mainNotebookDir = new File(mainNotePath);
mainNotebookDir.mkdirs();
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo");
conf = ZeppelinConfiguration.create();
this.schedulerFactory = new SchedulerFactory();
this.schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
super.setUp();
this.schedulerFactory = SchedulerFactory.singleton();
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search,
notebookAuthorization, null);
}
@After
public void tearDown() throws Exception {
if (!FileUtils.deleteQuietly(mainZepDir)) {
LOG.error("Failed to delete {} ", mainZepDir.getName());
if (!FileUtils.deleteQuietly(testRootDir)) {
LOG.error("Failed to delete {} ", testRootDir.getName());
}
}
@ -120,7 +84,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
int numNotes = notebookRepo.list(null).size();
// when create invalid json file
File testNoteDir = new File(mainNotebookDir, "test");
File testNoteDir = new File(notebookDir, "interpreter/test");
testNoteDir.mkdir();
FileUtils.writeStringToFile(new File(testNoteDir, "note.json"), "");
@ -132,7 +96,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
public void testSaveNotebook() throws IOException, InterruptedException {
AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p1.getConfig();

93
zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
View file

@ -17,35 +17,27 @@
package org.apache.zeppelin.resource;
import com.google.gson.Gson;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
/**
* Unittest for DistributedResourcePool
*/
public class DistributedResourcePoolTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup1;
private InterpreterGroup intpGroup2;
private HashMap<String, String> env;
public class DistributedResourcePoolTest extends AbstractInterpreterTest {
private RemoteInterpreter intp1;
private RemoteInterpreter intp2;
private InterpreterContext context;
@ -55,50 +47,10 @@ public class DistributedResourcePoolTest {
@Before
public void setUp() throws Exception {
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
Properties p = new Properties();
intp1 = new RemoteInterpreter(
p,
"note",
MockInterpreterResourcePool.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false
);
intpGroup1 = new InterpreterGroup("intpGroup1");
intpGroup1.put("note", new LinkedList<Interpreter>());
intpGroup1.get("note").add(intp1);
intp1.setInterpreterGroup(intpGroup1);
intp2 = new RemoteInterpreter(
p,
"note",
MockInterpreterResourcePool.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false
);
intpGroup2 = new InterpreterGroup("intpGroup2");
intpGroup2.put("note", new LinkedList<Interpreter>());
intpGroup2.get("note").add(intp2);
intp2.setInterpreterGroup(intpGroup2);
super.setUp();
InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("mock_resource_pool");
intp1 = (RemoteInterpreter) interpreterSetting.getInterpreter("user1", "note1", "mock_resource_pool");
intp2 = (RemoteInterpreter) interpreterSetting.getInterpreter("user2", "note1", "mock_resource_pool");
context = new InterpreterContext(
"note",
@ -117,26 +69,13 @@ public class DistributedResourcePoolTest {
intp1.open();
intp2.open();
eventPoller1 = new RemoteInterpreterEventPoller(null, null);
eventPoller1.setInterpreterGroup(intpGroup1);
eventPoller1.setInterpreterProcess(intpGroup1.getRemoteInterpreterProcess());
eventPoller2 = new RemoteInterpreterEventPoller(null, null);
eventPoller2.setInterpreterGroup(intpGroup2);
eventPoller2.setInterpreterProcess(intpGroup2.getRemoteInterpreterProcess());
eventPoller1.start();
eventPoller2.start();
eventPoller1 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller();
eventPoller2 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller();
}
@After
public void tearDown() throws Exception {
eventPoller1.shutdown();
intp1.close();
intpGroup1.close();
eventPoller2.shutdown();
intp2.close();
intpGroup2.close();
interpreterSettingManager.close();
}
@Test
@ -235,13 +174,13 @@ public class DistributedResourcePoolTest {
// then get all resources.
assertEquals(4, ResourcePoolUtils.getAllResources().size());
assertEquals(4, interpreterSettingManager.getAllResources().size());
// when remove all resources from note1
ResourcePoolUtils.removeResourcesBelongsToNote("note1");
interpreterSettingManager.removeResourcesBelongsToNote("note1");
// then resources should be removed.
assertEquals(2, ResourcePoolUtils.getAllResources().size());
assertEquals(2, interpreterSettingManager.getAllResources().size());
assertEquals("", gson.fromJson(
intp1.interpret("get note1:paragraph1:key1", context).message().get(0).getData(),
String.class));
@ -251,10 +190,10 @@ public class DistributedResourcePoolTest {
// when remove all resources from note2:paragraph1
ResourcePoolUtils.removeResourcesBelongsToParagraph("note2", "paragraph1");
interpreterSettingManager.removeResourcesBelongsToParagraph("note2", "paragraph1");
// then 1
assertEquals(1, ResourcePoolUtils.getAllResources().size());
assertEquals(1, interpreterSettingManager.getAllResources().size());
assertEquals("value2", gson.fromJson(
intp1.interpret("get note2:paragraph2:key2", context).message().get(0).getData(),
String.class));

131
zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
View file

@ -17,83 +17,84 @@
package org.apache.zeppelin.scheduler;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterRunner;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterSetting interpreterSetting;
private SchedulerFactory schedulerSvc;
private static final int TICK_WAIT = 100;
private static final int MAX_WAIT_CYCLES = 100;
@Before
public void setUp() throws Exception{
public void setUp() throws Exception {
schedulerSvc = new SchedulerFactory();
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterA.class.getName(), "mock", true, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
interpreterSetting = new InterpreterSetting.Builder()
.setId("test")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.setRunner(runner)
.setInterpreterDir("../interpeters/test")
.create();
}
@After
public void tearDown(){
public void tearDown() {
interpreterSetting.close();
}
@Test
public void test() throws Exception {
Properties p = new Properties();
final InterpreterGroup intpGroup = new InterpreterGroup();
Map<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(
p,
"note",
MockInterpreterA.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
this,
null,
"anonymous",
false);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
intpA.open();
Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
intpA.getInterpreterProcess(),
10);
Scheduler scheduler = intpA.getScheduler();
Job job = new Job("jobId", "jobName", null, 200) {
Object results;
@Override
public Object getReturn() {
return results;
@ -120,7 +121,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
return "1000";
@ -145,7 +146,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
}
assertTrue(job.isRunning());
Thread.sleep(5*TICK_WAIT);
Thread.sleep(5 * TICK_WAIT);
assertEquals(0, scheduler.getJobsWaiting().size());
assertEquals(1, scheduler.getJobsRunning().size());
@ -165,34 +166,10 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
@Test
public void testAbortOnPending() throws Exception {
Properties p = new Properties();
final InterpreterGroup intpGroup = new InterpreterGroup();
Map<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(
p,
"note",
MockInterpreterA.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
this,
null,
"anonymous",
false);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
intpA.open();
Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
intpA.getInterpreterProcess(),
10);
Scheduler scheduler = intpA.getScheduler();
Job job1 = new Job("jobId1", "jobName1", null, 200) {
Object results;
@ -205,7 +182,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
@ -255,7 +232,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
@ -358,7 +335,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
}
@Override
public void onParaInfosReceived(String noteId, String paragraphId,
String interpreterSettingId, Map<String, String> metaInfos) {
public void onParaInfosReceived(String noteId, String paragraphId,
String interpreterSettingId, Map<String, String> metaInfos) {
}
}

115
zeppelin-zengine/src/test/resources/conf/interpreter.json Normal file
View file

@ -0,0 +1,115 @@
{
"interpreterSettings": {
"2C3RWCVAG": {
"id": "2C3RWCVAG",
"name": "test",
"group": "test",
"properties": {
"property_1": "value_1",
"property_2": "new_value_2",
"property_3": "value_3"
},
"status": "READY",
"interpreterGroup": [
{
"name": "echo",
"class": "org.apache.zeppelin.interpreter.EchoInterpreter",
"defaultInterpreter": true,
"editor": {
"language": "java",
"editOnDblClick": false
}
}
],
"dependencies": [],
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
},
"2CKWE7B19": {
"id": "2CKWE7B19",
"name": "test2",
"group": "test",
"properties": {
"property_1": "value_1",
"property_2": "new_value_2",
"property_3": "value_3"
},
"status": "READY",
"interpreterGroup": [
{
"name": "echo",
"class": "org.apache.zeppelin.interpreter.EchoInterpreter",
"defaultInterpreter": true,
"editor": {
"language": "java",
"editOnDblClick": false
}
}
],
"dependencies": [],
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
},
"interpreterBindings": {
"2C6793KRV": [
"2C48Y7FSJ",
"2C63XW4XE",
"2C66GE1VB",
"2C5VH924X",
"2C4BJDRRZ",
"2C3SQSB7V",
"2C4HKDCQW",
"2C3DR183X",
"2C66Z9XPQ",
"2C3PTPMUH",
"2C69WE69N",
"2C5SRRXHM",
"2C4ZD49PF",
"2C6V3D44K",
"2C4UB1UZA",
"2C5S1R21W",
"2C5DCRVGM",
"2C686X8ZH",
"2C3RWCVAG",
"2C3JKFMJU",
"2C3VECEG2"
]
},
"interpreterRepositories": [
{
"id": "central",
"type": "default",
"url": "http://repo1.maven.org/maven2/",
"releasePolicy": {
"enabled": true,
"updatePolicy": "daily",
"checksumPolicy": "warn"
},
"snapshotPolicy": {
"enabled": true,
"updatePolicy": "daily",
"checksumPolicy": "warn"
},
"mirroredRepositories": [],
"repositoryManager": false
}
]
}

12
zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json
View file

@ -1,12 +0,0 @@
[
{
"group": "mock11",
"name": "mock11",
"className": "org.apache.zeppelin.interpreter.mock.MockInterpreter11",
"properties": {
},
"editor": {
"language": "java"
}
}
]

19
zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json Normal file
View file

@ -0,0 +1,19 @@
[
{
"group": "mock1",
"name": "mock1",
"className": "org.apache.zeppelin.interpreter.mock.MockInterpreter1",
"properties": {
},
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
]

19
zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json Normal file
View file

@ -0,0 +1,19 @@
[
{
"group": "mock2",
"name": "mock2",
"className": "org.apache.zeppelin.interpreter.mock.MockInterpreter2",
"properties": {
},
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "isolated",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
]

19
zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json Normal file
View file

@ -0,0 +1,19 @@
[
{
"group": "mock_resource_pool",
"name": "mock_resource_pool",
"className": "org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool",
"properties": {
},
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
]

42
zeppelin-zengine/src/test/resources/interpreter/test/interpreter-setting.json Normal file
View file

@ -0,0 +1,42 @@
[
{
"group": "test",
"name": "double_echo",
"className": "org.apache.zeppelin.interpreter.DoubleEchoInterpreter",
"properties": {
"property_1": {
"envName": "PROPERTY_1",
"propertyName": "property_1",
"defaultValue": "value_1",
"description": "desc_1"
},
"property_2": {
"envName": "PROPERTY_2",
"propertyName": "property_2",
"defaultValue": "value_2",
"description": "desc_2"
}
}
},
{
"group": "test",
"name": "echo",
"defaultInterpreter": true,
"className": "org.apache.zeppelin.interpreter.EchoInterpreter",
"properties": {
"property_1": {
"envName": "PROPERTY_1",
"propertyName": "property_1",
"defaultValue": "value_1",
"description": "desc_1"
},
"property_2": {
"envName": "PROPERTY_2",
"propertyName": "property_2",
"defaultValue": "value_2",
"description": "desc_2"
}
}
}
]

3
zeppelin-zengine/src/test/resources/log4j.properties
View file

@ -35,7 +35,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
log4j.logger.org.apache.zeppelin.scheduler=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN
@ -45,4 +44,6 @@ log4j.logger.DataNucleus.Datastore=ERROR
# Log all JDBC parameters
log4j.logger.org.hibernate.type=ALL
log4j.logger.org.apache.zeppelin.interpreter=DEBUG
log4j.logger.org.apache.zeppelin.scheduler=DEBUG