Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[ZEPPELIN-2627] Interpreter Component Refactoring

Browse source
This commit is contained in:
Jeff Zhang 2017-05-23 15:52:15 +08:00
parent 30bfcae0c0
commit 4724c98611
96 changed files with 5259 additions and 6427 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.travis.yml
View file

@ -162,6 +162,7 @@ after_success:
after_failure:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
- find . -name rat.txt | xargs cat
- cat logs/*
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log

8
helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
View file

@ -43,13 +43,13 @@ public class ZeppelinDevServer extends
}
@Override
protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
protected Interpreter getInterpreter(String sessionId, String className) throws TException {
synchronized (this) {
InterpreterGroup interpreterGroup = getInterpreterGroup();
if (interpreterGroup == null || interpreterGroup.isEmpty()) {
createInterpreter(
"dev",
sessionKey,
sessionId,
DevInterpreter.class.getName(),
new HashMap<String, String>(),
"anonymous");
@ -57,11 +57,11 @@ public class ZeppelinDevServer extends
}
}
Interpreter intp = super.getInterpreter(sessionKey, className);
Interpreter intp = super.getInterpreter(sessionId, className);
interpreter = (DevInterpreter) (
((LazyOpenInterpreter) intp).getInnerInterpreter());
interpreter.setInterpreterEvent(this);
return super.getInterpreter(sessionKey, className);
return super.getInterpreter(sessionId, className);
}
@Override

22
zeppelin-interpreter/pom.xml
View file

@ -51,6 +51,11 @@
<dependencies>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
@ -231,4 +236,21 @@
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

22
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -17,6 +17,14 @@
package org.apache.zeppelin.conf;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;
import org.apache.commons.configuration.tree.ConfigurationNode;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.URL;
import java.util.Arrays;
@ -24,15 +32,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.XMLConfiguration;
import org.apache.commons.configuration.tree.ConfigurationNode;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.notebook.repo.GitNotebookRepo;
import org.apache.zeppelin.util.Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Zeppelin configuration.
*
@ -528,7 +527,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
for (ZeppelinConfiguration.ConfVars v : ZeppelinConfiguration.ConfVars.values()) {
for (ConfVars v : ConfVars.values()) {
String key = v.getVarName();
if (!predicate.apply(key)) {
@ -653,7 +652,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_MONGO_COLLECTION("zeppelin.notebook.mongo.collection", "notes"),
ZEPPELIN_NOTEBOOK_MONGO_URI("zeppelin.notebook.mongo.uri", "mongodb://localhost"),
ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT("zeppelin.notebook.mongo.autoimport", false),
ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", GitNotebookRepo.class.getName()),
ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage",
"org.apache.zeppelin.notebook.repo.GitNotebookRepo"),
ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC("zeppelin.notebook.one.way.sync", false),
// whether by default note is public or private
ZEPPELIN_NOTEBOOK_PUBLIC("zeppelin.notebook.public", true),

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
View file

@ -149,7 +149,6 @@ public abstract class Interpreter {
@ZeppelinApi
public Interpreter(Properties property) {
logger.debug("Properties: {}", property);
this.property = property;
}

115
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java Normal file
View file

@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* //TODO(zjffdu) considering to move to InterpreterSettingManager
*
* Manage interpreters.
*/
public class InterpreterFactory {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterFactory.class);
private final InterpreterSettingManager interpreterSettingManager;
public InterpreterFactory(InterpreterSettingManager interpreterSettingManager) {
this.interpreterSettingManager = interpreterSettingManager;
}
private InterpreterSetting getInterpreterSettingByGroup(List<InterpreterSetting> settings,
String group) {
Preconditions.checkNotNull(group, "group should be not null");
for (InterpreterSetting setting : settings) {
if (group.equals(setting.getName())) {
return setting;
}
}
return null;
}
public Interpreter getInterpreter(String user, String noteId, String replName) {
List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(noteId);
InterpreterSetting setting;
Interpreter interpreter;
if (settings == null || settings.size() == 0) {
LOGGER.error("No interpreter is binded to this note: " + noteId);
return null;
}
if (StringUtils.isBlank(replName)) {
// Get the default interpreter of the first interpreter binding
InterpreterSetting defaultSetting = settings.get(0);
return defaultSetting.getDefaultInterpreter(user, noteId);
}
String[] replNameSplit = replName.split("\\.");
if (replNameSplit.length == 2) {
String group = replNameSplit[0];
String name = replNameSplit[1];
setting = getInterpreterSettingByGroup(settings, group);
if (null != setting) {
interpreter = setting.getInterpreter(user, noteId, name);
if (null != interpreter) {
return interpreter;
}
}
throw new InterpreterException(replName + " interpreter not found");
} else {
// first assume replName is 'name' of interpreter. ('groupName' is ommitted)
// search 'name' from first (default) interpreter group
// TODO(jl): Handle with noteId to support defaultInterpreter per note.
setting = settings.get(0);
interpreter = setting.getInterpreter(user, noteId, replName);
if (null != interpreter) {
return interpreter;
}
// next, assume replName is 'group' of interpreter ('name' is ommitted)
// search interpreter group and return first interpreter.
setting = getInterpreterSettingByGroup(settings, replName);
if (null != setting) {
return setting.getDefaultInterpreter(user, noteId);
}
// Support the legacy way to use it
for (InterpreterSetting s : settings) {
if (s.getGroup().equals(replName)) {
return setting.getDefaultInterpreter(user, noteId);
}
}
}
//TODO(zjffdu) throw InterpreterException instead of return null
return null;
}
}

274
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -17,14 +17,7 @@
package org.apache.zeppelin.interpreter;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import com.google.common.annotations.VisibleForTesting;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.resource.ResourcePool;
@ -33,91 +26,93 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
/**
* InterpreterGroup is list of interpreters in the same interpreter group.
* For example spark, pyspark, sql interpreters are in the same 'spark' group
* and InterpreterGroup will have reference to these all interpreters.
* InterpreterGroup is collections of interpreter sessions.
* One session could include multiple interpreters.
* For example spark, pyspark, sql interpreters are in the same 'spark' interpreter session.
*
* Remember, list of interpreters are dedicated to a session. Session could be shared across user
* or notes, so the sessionId could be user or noteId or their combination.
* So InterpreterGroup internally manages map of [interpreterSessionKey(noteId, user, or
* So InterpreterGroup internally manages map of [sessionId(noteId, user, or
* their combination), list of interpreters]
*
* A InterpreterGroup runs on interpreter process.
* And unit of interpreter instantiate, restart, bind, unbind.
* A InterpreterGroup runs both in zeppelin server process and interpreter process.
*/
public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
String id;
public class InterpreterGroup {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroup.class);
AngularObjectRegistry angularObjectRegistry;
InterpreterHookRegistry hookRegistry;
RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
ResourcePool resourcePool;
boolean angularRegistryPushed = false;
// map [notebook session, Interpreters in the group], to support per note session interpreters
//Map<String, List<Interpreter>> interpreters = new ConcurrentHashMap<String,
// List<Interpreter>>();
private static final Map<String, InterpreterGroup> allInterpreterGroups =
new ConcurrentHashMap<>();
public static InterpreterGroup getByInterpreterGroupId(String id) {
return allInterpreterGroups.get(id);
}
public static Collection<InterpreterGroup> getAll() {
return new LinkedList(allInterpreterGroups.values());
}
private String id;
// sessionId --> interpreters
private Map<String, List<Interpreter>> sessions = new ConcurrentHashMap();
private InterpreterSetting interpreterSetting;
private AngularObjectRegistry angularObjectRegistry;
private InterpreterHookRegistry hookRegistry;
private RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
private ResourcePool resourcePool;
private boolean angularRegistryPushed = false;
/**
* Create InterpreterGroup with given id
* Create InterpreterGroup with given id, used in InterpreterProcess
* @param id
*/
public InterpreterGroup(String id) {
this.id = id;
allInterpreterGroups.put(id, this);
}
/**
* Create InterpreterGroup with given id and interpreterSetting, used in ZeppelinServer
* @param id
* @param interpreterSetting
*/
InterpreterGroup(String id, InterpreterSetting interpreterSetting) {
this.id = id;
this.interpreterSetting = interpreterSetting;
}
/**
* Create InterpreterGroup with autogenerated id
*/
@VisibleForTesting
public InterpreterGroup() {
getId();
allInterpreterGroups.put(id, this);
this.id = generateId();
}
private static String generateId() {
return "InterpreterGroup_" + System.currentTimeMillis() + "_"
+ new Random().nextInt();
return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new Random().nextInt();
}
public String getId() {
synchronized (this) {
if (id == null) {
id = generateId();
}
return id;
}
return this.id;
}
/**
* Get combined property of all interpreters in this group
* @return
*/
public Properties getProperty() {
Properties p = new Properties();
//TODO(zjffdu) change it to getSession. For now just keep this method to reduce code change
public synchronized List<Interpreter> get(String sessionId) {
return sessions.get(sessionId);
}
for (List<Interpreter> intpGroupForASession : this.values()) {
for (Interpreter intp : intpGroupForASession) {
p.putAll(intp.getProperty());
}
// it's okay to break here while every List<Interpreters> will have the same property set
break;
//TODO(zjffdu) change it to addSession. For now just keep this method to reduce code change
public synchronized void put(String sessionId, List<Interpreter> interpreters) {
this.sessions.put(sessionId, interpreters);
}
public synchronized void addInterpreterToSession(Interpreter interpreter, String sessionId) {
LOGGER.debug("Add Interpreter {} to session {}", interpreter.getClassName(), sessionId);
List<Interpreter> interpreters = get(sessionId);
if (interpreters == null) {
interpreters = new ArrayList<>();
}
return p;
interpreters.add(interpreter);
put(sessionId, interpreters);
}
//TODO(zjffdu) rename it to a more proper name.
//For now just keep this method to reduce code change
public Collection<List<Interpreter>> values() {
return sessions.values();
}
public AngularObjectRegistry getAngularObjectRegistry() {
@ -136,128 +131,69 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
this.hookRegistry = hookRegistry;
}
public InterpreterSetting getInterpreterSetting() {
return interpreterSetting;
}
public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
if (remoteInterpreterProcess == null) {
LOGGER.info("Create InterperterProcess for InterpreterGroup: " + getId());
remoteInterpreterProcess = interpreterSetting.createInterpreterProcess();
}
return remoteInterpreterProcess;
}
public RemoteInterpreterProcess getRemoteInterpreterProcess() {
return remoteInterpreterProcess;
}
public void setRemoteInterpreterProcess(RemoteInterpreterProcess remoteInterpreterProcess) {
this.remoteInterpreterProcess = remoteInterpreterProcess;
}
/**
* Close all interpreter instances in this group
*/
public void close() {
LOGGER.info("Close interpreter group " + getId());
List<Interpreter> intpToClose = new LinkedList<>();
for (List<Interpreter> intpGroupForSession : this.values()) {
intpToClose.addAll(intpGroupForSession);
public synchronized void close() {
LOGGER.info("Close InterpreterGroup: " + id);
for (String sessionId : sessions.keySet()) {
close(sessionId);
}
close(intpToClose);
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);
}
/**
* Close all interpreter instances in this group for the session
* Close all interpreter instances in this session
* @param sessionId
*/
public void close(String sessionId) {
LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
final List<Interpreter> intpForSession = this.get(sessionId);
close(intpForSession);
public synchronized void close(String sessionId) {
LOGGER.info("Close Session: " + sessionId);
close(sessions.remove(sessionId));
//TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server
if (sessions.isEmpty() && interpreterSetting != null) {
LOGGER.info("Remove this InterpreterGroup {} as all the sessions are closed", id);
interpreterSetting.removeInterpreterGroup(id);
if (remoteInterpreterProcess != null) {
LOGGER.info("Kill RemoteInterpreterProcess");
remoteInterpreterProcess.stop();
remoteInterpreterProcess = null;
}
}
}
private void close(final Collection<Interpreter> intpToClose) {
close(null, null, null, intpToClose);
public int getSessionNum() {
return sessions.size();
}
public void close(final Map<String, InterpreterGroup> interpreterGroupRef,
final String processKey, final String sessionKey) {
LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionKey);
close(interpreterGroupRef, processKey, sessionKey, this.get(sessionKey));
}
private void close(final Map<String, InterpreterGroup> interpreterGroupRef,
final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
if (intpToClose == null) {
private void close(Collection<Interpreter> interpreters) {
if (interpreters == null) {
return;
}
Thread t = new Thread() {
public void run() {
for (Interpreter interpreter : intpToClose) {
Scheduler scheduler = interpreter.getScheduler();
interpreter.close();
if (null != scheduler) {
SchedulerFactory.singleton().removeScheduler(scheduler.getName());
}
}
if (remoteInterpreterProcess != null) {
//TODO(jl): Because interpreter.close() runs as a seprate thread, we cannot guarantee
// refernceCount is a proper value. And as the same reason, we must not call
// remoteInterpreterProcess.dereference twice - this method also be called by
// interpreter.close().
// remoteInterpreterProcess.dereference();
if (remoteInterpreterProcess.referenceCount() <= 0) {
remoteInterpreterProcess = null;
allInterpreterGroups.remove(id);
}
}
// TODO(jl): While closing interpreters in a same session, we should remove after all
// interpreters are removed. OMG. It's too dirty!!
if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
interpreterGroupRef.remove(processKey);
} else {
interpreterGroup.remove(sessionKey);
}
}
for (Interpreter interpreter : interpreters) {
Scheduler scheduler = interpreter.getScheduler();
interpreter.close();
//TODO(zjffdu) move the close of schedule to Interpreter
if (null != scheduler) {
SchedulerFactory.singleton().removeScheduler(scheduler.getName());
}
};
t.start();
try {
t.join();
} catch (InterruptedException e) {
LOGGER.error("Can't close interpreter: {}", getId(), e);
}
}
/**
* Close all interpreter instances in this group
*/
public void shutdown() {
LOGGER.info("Close interpreter group " + getId());
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);
List<Interpreter> intpToClose = new LinkedList<>();
for (List<Interpreter> intpGroupForSession : this.values()) {
intpToClose.addAll(intpGroupForSession);
}
close(intpToClose);
}
public void setResourcePool(ResourcePool resourcePool) {
@ -275,4 +211,22 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
public void setAngularRegistryPushed(boolean angularRegistryPushed) {
this.angularRegistryPushed = angularRegistryPushed;
}
public synchronized List<Interpreter> getOrCreateSession(String user, String sessionId) {
if (sessions.containsKey(sessionId)) {
return sessions.get(sessionId);
} else {
List<Interpreter> interpreters = interpreterSetting.createInterpreters(user, sessionId);
for (Interpreter interpreter : interpreters) {
interpreter.setInterpreterGroup(this);
}
LOGGER.info("Create Session {} in InterpreterGroup {} for user {}", sessionId, id, user);
sessions.put(sessionId, interpreters);
return interpreters;
}
}
public boolean isEmpty() {
return sessions.isEmpty();
}
}

0
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
View file

101
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java Normal file
View file

@ -0,0 +1,101 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.internal.StringMap;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.repository.RemoteRepository;
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.PosixFilePermission;
import java.util.*;
import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
/**
*
*/
public class InterpreterInfoSaving {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterInfoSaving.class);
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
public Map<String, InterpreterSetting> interpreterSettings = new HashMap<>();
public Map<String, List<String>> interpreterBindings = new HashMap<>();
public List<RemoteRepository> interpreterRepositories = new ArrayList<>();
public static InterpreterInfoSaving loadFromFile(Path file) throws IOException {
LOGGER.info("Load interpreter setting from file: " + file);
InterpreterInfoSaving infoSaving = null;
try (BufferedReader json = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
JsonParser jsonParser = new JsonParser();
JsonObject jsonObject = jsonParser.parse(json).getAsJsonObject();
infoSaving = InterpreterInfoSaving.fromJson(jsonObject.toString());
if (infoSaving != null && infoSaving.interpreterSettings != null) {
for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) {
// Always use separate interpreter process
// While we decided to turn this feature on always (without providing
// enable/disable option on GUI).
// previously created setting should turn this feature on here.
interpreterSetting.getOption().setRemote(true);
interpreterSetting.convertPermissionsFromUsersToOwners(
jsonObject.getAsJsonObject("interpreterSettings")
.getAsJsonObject(interpreterSetting.getId()));
}
}
}
return infoSaving == null ? new InterpreterInfoSaving() : infoSaving;
}
public void saveToFile(Path file) throws IOException {
if (!Files.exists(file)) {
Files.createFile(file);
try {
Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
Files.setPosixFilePermissions(file, permissions);
} catch (UnsupportedOperationException e) {
// File system does not support Posix file permissions (likely windows) - continue anyway.
LOGGER.warn("unable to setPosixFilePermissions on '{}'.", file);
};
}
LOGGER.info("Save Interpreter Settings to " + file);
IOUtils.write(this.toJson(), new FileOutputStream(file.toFile()));
}
public String toJson() {
return gson.toJson(this);
}
public static InterpreterInfoSaving fromJson(String json) {
return gson.fromJson(json, InterpreterInfoSaving.class);
}
}

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
View file

@ -34,6 +34,7 @@ public class InterpreterProperty {
public InterpreterProperty(String name, Object value) {
this.name = name;
this.value = value;
this.type = "textarea";
}
public String getName() {

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
View file

@ -1,5 +1,6 @@
package org.apache.zeppelin.interpreter;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.annotations.SerializedName;
/**
@ -12,6 +13,16 @@ public class InterpreterRunner {
@SerializedName("win")
private String winPath;
public InterpreterRunner() {
}
@VisibleForTesting
public InterpreterRunner(String linuxPath, String winPath) {
this.linuxPath = linuxPath;
this.winPath = winPath;
}
public String getPath() {
return System.getProperty("os.name").startsWith("Windows") ? winPath : linuxPath;
}

911
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java Normal file
View file

@ -0,0 +1,911 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.annotations.SerializedName;
import com.google.gson.internal.StringMap;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE;
import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
import static org.apache.zeppelin.util.IdHashes.generateId;
/**
* Represent one InterpreterSetting in the interpreter setting page
*/
public class InterpreterSetting {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSetting.class);
private static final String SHARED_PROCESS = "shared_process";
private static final String SHARED_SESSION = "shared_session";
private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
"language", (Object) "text",
"editOnDblClick", false);
private String id;
private String name;
// the original interpreter setting template name where it is created from
private String group;
//TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json
/**
* properties can be either Properties or Map<String, InterpreterProperty>
* properties should be:
* - Properties when Interpreter instances are saved to `conf/interpreter.json` file
* - Map<String, InterpreterProperty> when Interpreters are registered
* : this is needed after https://github.com/apache/zeppelin/pull/1145
* which changed the way of getting default interpreter setting AKA interpreterSettingsRef
* Note(mina): In order to simplify the implementation, I chose to change properties
* from Properties to Object instead of creating new classes.
*/
private Object properties = new Properties();
private Status status;
private String errorReason;
@SerializedName("interpreterGroup")
private List<InterpreterInfo> interpreterInfos;
private List<Dependency> dependencies = new ArrayList<>();
private InterpreterOption option = new InterpreterOption(true);
@SerializedName("runner")
private InterpreterRunner interpreterRunner;
///////////////////////////////////////////////////////////////////////////////////////////
private transient InterpreterSettingManager interpreterSettingManager;
private transient String interpreterDir;
private final transient Map<String, InterpreterGroup> interpreterGroups =
new ConcurrentHashMap<>();
private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock;
private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock;
private transient AngularObjectRegistryListener angularObjectRegistryListener;
private transient RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private transient ApplicationEventListener appEventListener;
private transient DependencyResolver dependencyResolver;
private transient Map<String, String> infos;
// Map of the note and paragraphs which has runtime infos generated by this interpreter setting.
// This map is used to clear the infos in paragraph when the interpretersetting is restarted
private transient Map<String, Set<String>> runtimeInfosToBeCleared;
private transient ZeppelinConfiguration conf = new ZeppelinConfiguration();
private transient Map<String, URLClassLoader> cleanCl =
Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
///////////////////////////////////////////////////////////////////////////////////////////
/**
* Builder class for InterpreterSetting
*/
public static class Builder {
private InterpreterSetting interpreterSetting;
public Builder() {
this.interpreterSetting = new InterpreterSetting();
}
public Builder setId(String id) {
interpreterSetting.id = id;
return this;
}
public Builder setName(String name) {
interpreterSetting.name = name;
return this;
}
public Builder setGroup(String group) {
interpreterSetting.group = group;
return this;
}
public Builder setInterpreterInfos(List<InterpreterInfo> interpreterInfos) {
interpreterSetting.interpreterInfos = interpreterInfos;
return this;
}
public Builder setProperties(Object properties) {
interpreterSetting.properties = properties;
return this;
}
public Builder setOption(InterpreterOption option) {
interpreterSetting.option = option;
return this;
}
public Builder setInterpreterDir(String interpreterDir) {
interpreterSetting.interpreterDir = interpreterDir;
return this;
}
public Builder setRunner(InterpreterRunner runner) {
interpreterSetting.interpreterRunner = runner;
return this;
}
public Builder setDependencies(List<Dependency> dependencies) {
interpreterSetting.dependencies = dependencies;
return this;
}
public Builder setConf(ZeppelinConfiguration conf) {
interpreterSetting.conf = conf;
return this;
}
public Builder setDependencyResolver(DependencyResolver dependencyResolver) {
interpreterSetting.dependencyResolver = dependencyResolver;
return this;
}
// public Builder setInterpreterRunner(InterpreterRunner runner) {
// interpreterSetting.interpreterRunner = runner;
// return this;
// }
public Builder setIntepreterSettingManager(
InterpreterSettingManager interpreterSettingManager) {
interpreterSetting.interpreterSettingManager = interpreterSettingManager;
return this;
}
public Builder setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener
remoteInterpreterProcessListener) {
interpreterSetting.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
return this;
}
public Builder setAngularObjectRegistryListener(
AngularObjectRegistryListener angularObjectRegistryListener) {
interpreterSetting.angularObjectRegistryListener = angularObjectRegistryListener;
return this;
}
public Builder setApplicationEventListener(ApplicationEventListener applicationEventListener) {
interpreterSetting.appEventListener = applicationEventListener;
return this;
}
public InterpreterSetting create() {
// post processing
interpreterSetting.postProcessing();
return interpreterSetting;
}
}
public InterpreterSetting() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.id = generateId();
interpreterGroupReadLock = lock.readLock();
interpreterGroupWriteLock = lock.writeLock();
}
void postProcessing() {
this.status = Status.READY;
}
/**
* Create interpreter from InterpreterSettingTemplate
*
* @param o interpreterSetting from InterpreterSettingTemplate
*/
public InterpreterSetting(InterpreterSetting o) {
this();
this.id = generateId();
this.name = o.name;
this.group = o.group;
this.properties = convertInterpreterProperties(
(Map<String, DefaultInterpreterProperty>) o.getProperties());
this.interpreterInfos = new ArrayList<>(o.getInterpreterInfos());
this.option = InterpreterOption.fromInterpreterOption(o.getOption());
this.dependencies = new ArrayList<>(o.getDependencies());
this.interpreterDir = o.getInterpreterDir();
this.interpreterRunner = o.getInterpreterRunner();
this.conf = o.getConf();
}
public AngularObjectRegistryListener getAngularObjectRegistryListener() {
return angularObjectRegistryListener;
}
public RemoteInterpreterProcessListener getRemoteInterpreterProcessListener() {
return remoteInterpreterProcessListener;
}
public ApplicationEventListener getAppEventListener() {
return appEventListener;
}
public DependencyResolver getDependencyResolver() {
return dependencyResolver;
}
public InterpreterSettingManager getInterpreterSettingManager() {
return interpreterSettingManager;
}
public void setAngularObjectRegistryListener(AngularObjectRegistryListener
angularObjectRegistryListener) {
this.angularObjectRegistryListener = angularObjectRegistryListener;
}
public void setAppEventListener(ApplicationEventListener appEventListener) {
this.appEventListener = appEventListener;
}
public void setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener
remoteInterpreterProcessListener) {
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
}
public void setDependencyResolver(DependencyResolver dependencyResolver) {
this.dependencyResolver = dependencyResolver;
}
public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) {
this.interpreterSettingManager = interpreterSettingManager;
}
public String getId() {
return id;
}
public String getName() {
return name;
}
public String getGroup() {
return group;
}
private String getInterpreterGroupId(String user, String noteId) {
String key;
if (option.isExistingProcess) {
key = Constants.EXISTING_PROCESS;
} else if (getOption().isProcess()) {
key = (option.perUserIsolated() ? user : "") + ":" + (option.perNoteIsolated() ? noteId : "");
} else {
key = SHARED_PROCESS;
}
//TODO(zjffdu) we encode interpreter setting id into groupId, this is not a good design
return id + ":" + key;
}
private String getInterpreterSessionId(String user, String noteId) {
String key;
if (option.isExistingProcess()) {
key = Constants.EXISTING_PROCESS;
} else if (option.perNoteScoped() && option.perUserScoped()) {
key = user + ":" + noteId;
} else if (option.perUserScoped()) {
key = user;
} else if (option.perNoteScoped()) {
key = noteId;
} else {
key = SHARED_SESSION;
}
return key;
}
public InterpreterGroup getOrCreateInterpreterGroup(String user, String noteId) {
String groupId = getInterpreterGroupId(user, noteId);
try {
interpreterGroupWriteLock.lock();
if (!interpreterGroups.containsKey(groupId)) {
LOGGER.info("Create InterpreterGroup with groupId {} for user {} and note {}",
groupId, user, noteId);
InterpreterGroup intpGroup = createInterpreterGroup(groupId);
interpreterGroups.put(groupId, intpGroup);
}
return interpreterGroups.get(groupId);
} finally {
interpreterGroupWriteLock.unlock();;
}
}
void removeInterpreterGroup(String groupId) {
this.interpreterGroups.remove(groupId);
}
InterpreterGroup getInterpreterGroup(String user, String noteId) {
String groupId = getInterpreterGroupId(user, noteId);
try {
interpreterGroupReadLock.lock();
return interpreterGroups.get(groupId);
} finally {
interpreterGroupReadLock.unlock();;
}
}
InterpreterGroup getInterpreterGroup(String groupId) {
return interpreterGroups.get(groupId);
}
@VisibleForTesting
public ArrayList<InterpreterGroup> getAllInterpreterGroups() {
try {
interpreterGroupReadLock.lock();
return new ArrayList(interpreterGroups.values());
} finally {
interpreterGroupReadLock.unlock();
}
}
Map<String, Object> getEditorFromSettingByClassName(String className) {
for (InterpreterInfo intpInfo : interpreterInfos) {
if (className.equals(intpInfo.getClassName())) {
if (intpInfo.getEditor() == null) {
break;
}
return intpInfo.getEditor();
}
}
return DEFAULT_EDITOR;
}
void closeInterpreters(String user, String noteId) {
InterpreterGroup interpreterGroup = getInterpreterGroup(user, noteId);
if (interpreterGroup != null) {
String sessionId = getInterpreterSessionId(user, noteId);
interpreterGroup.close(sessionId);
}
}
public void close() {
LOGGER.info("Close InterpreterSetting: " + name);
for (InterpreterGroup intpGroup : interpreterGroups.values()) {
intpGroup.close();
}
interpreterGroups.clear();
this.runtimeInfosToBeCleared = null;
this.infos = null;
}
public void setProperties(Object object) {
if (object instanceof StringMap) {
StringMap<String> map = (StringMap) properties;
Properties newProperties = new Properties();
for (String key : map.keySet()) {
newProperties.put(key, map.get(key));
}
this.properties = newProperties;
} else {
this.properties = object;
}
}
public Object getProperties() {
return properties;
}
@VisibleForTesting
public void setProperty(String name, String value) {
((Map<String, InterpreterProperty>) properties).put(name, new InterpreterProperty(name, value));
}
// This method is supposed to be only called by InterpreterSetting
// but not InterpreterSetting Template
public Properties getJavaProperties() {
Properties jProperties = new Properties();
Map<String, InterpreterProperty> iProperties = (Map<String, InterpreterProperty>) properties;
for (Map.Entry<String, InterpreterProperty> entry : iProperties.entrySet()) {
jProperties.setProperty(entry.getKey(), entry.getValue().getValue().toString());
}
if (!jProperties.containsKey("zeppelin.interpreter.output.limit")) {
jProperties.setProperty("zeppelin.interpreter.output.limit",
conf.getInt(ZEPPELIN_INTERPRETER_OUTPUT_LIMIT) + "");
}
if (!jProperties.containsKey("zeppelin.interpreter.max.poolsize")) {
jProperties.setProperty("zeppelin.interpreter.max.poolsize",
conf.getInt(ZEPPELIN_INTERPRETER_MAX_POOL_SIZE) + "");
}
String interpreterLocalRepoPath = conf.getInterpreterLocalRepoPath();
//TODO(zjffdu) change it to interpreterDir/{interpreter_name}
jProperties.setProperty("zeppelin.interpreter.localRepo",
interpreterLocalRepoPath + "/" + id);
return jProperties;
}
public ZeppelinConfiguration getConf() {
return conf;
}
public void setConf(ZeppelinConfiguration conf) {
this.conf = conf;
}
public List<Dependency> getDependencies() {
return dependencies;
}
public void setDependencies(List<Dependency> dependencies) {
this.dependencies = dependencies;
loadInterpreterDependencies();
}
public InterpreterOption getOption() {
return option;
}
public void setOption(InterpreterOption option) {
this.option = option;
}
public String getInterpreterDir() {
return interpreterDir;
}
public void setInterpreterDir(String interpreterDir) {
this.interpreterDir = interpreterDir;
}
public List<InterpreterInfo> getInterpreterInfos() {
return interpreterInfos;
}
void appendDependencies(List<Dependency> dependencies) {
for (Dependency dependency : dependencies) {
if (!this.dependencies.contains(dependency)) {
this.dependencies.add(dependency);
}
}
loadInterpreterDependencies();
}
void setInterpreterOption(InterpreterOption interpreterOption) {
this.option = interpreterOption;
}
public void setProperties(Properties p) {
this.properties = p;
}
void setGroup(String group) {
this.group = group;
}
void setName(String name) {
this.name = name;
}
/***
* Interpreter status
*/
public enum Status {
DOWNLOADING_DEPENDENCIES,
ERROR,
READY
}
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
public String getErrorReason() {
return errorReason;
}
public void setErrorReason(String errorReason) {
this.errorReason = errorReason;
}
public void setInterpreterInfos(List<InterpreterInfo> interpreterInfos) {
this.interpreterInfos = interpreterInfos;
}
public void setInfos(Map<String, String> infos) {
this.infos = infos;
}
public Map<String, String> getInfos() {
return infos;
}
public InterpreterRunner getInterpreterRunner() {
return interpreterRunner;
}
public void setInterpreterRunner(InterpreterRunner interpreterRunner) {
this.interpreterRunner = interpreterRunner;
}
public void addNoteToPara(String noteId, String paraId) {
if (runtimeInfosToBeCleared == null) {
runtimeInfosToBeCleared = new HashMap<>();
}
Set<String> paraIdSet = runtimeInfosToBeCleared.get(noteId);
if (paraIdSet == null) {
paraIdSet = new HashSet<>();
runtimeInfosToBeCleared.put(noteId, paraIdSet);
}
paraIdSet.add(paraId);
}
public Map<String, Set<String>> getNoteIdAndParaMap() {
return runtimeInfosToBeCleared;
}
public void clearNoteIdAndParaMap() {
runtimeInfosToBeCleared = null;
}
//////////////////////////// IMPORTANT ////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////
// This is the only place to create interpreters. For now we always create multiple interpreter
// together (one session). We don't support to create single interpreter yet.
List<Interpreter> createInterpreters(String user, String sessionId) {
List<Interpreter> interpreters = new ArrayList<>();
List<InterpreterInfo> interpreterInfos = getInterpreterInfos();
for (InterpreterInfo info : interpreterInfos) {
Interpreter interpreter = null;
if (option.isRemote()) {
interpreter = new RemoteInterpreter(getJavaProperties(), sessionId,
info.getClassName(), user);
} else {
interpreter = createLocalInterpreter(info.getClassName());
}
if (info.isDefaultInterpreter()) {
interpreters.add(0, interpreter);
} else {
interpreters.add(interpreter);
}
LOGGER.info("Interpreter {} created for user: {}, sessionId: {}",
interpreter.getClassName(), user, sessionId);
}
return interpreters;
}
// Create Interpreter in ZeppelinServer for non-remote mode
private Interpreter createLocalInterpreter(String className)
throws InterpreterException {
LOGGER.info("Create Local Interpreter {} from {}", className, interpreterDir);
ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
try {
URLClassLoader ccl = cleanCl.get(interpreterDir);
if (ccl == null) {
// classloader fallback
ccl = URLClassLoader.newInstance(new URL[]{}, oldcl);
}
boolean separateCL = true;
try { // check if server's classloader has driver already.
Class cls = this.getClass().forName(className);
if (cls != null) {
separateCL = false;
}
} catch (Exception e) {
LOGGER.error("exception checking server classloader driver", e);
}
URLClassLoader cl;
if (separateCL == true) {
cl = URLClassLoader.newInstance(new URL[]{}, ccl);
} else {
cl = ccl;
}
Thread.currentThread().setContextClassLoader(cl);
Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className);
Constructor<Interpreter> constructor =
replClass.getConstructor(new Class[]{Properties.class});
Interpreter repl = constructor.newInstance(getJavaProperties());
repl.setClassloaderUrls(ccl.getURLs());
LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl));
return intp;
} catch (SecurityException e) {
throw new InterpreterException(e);
} catch (NoSuchMethodException e) {
throw new InterpreterException(e);
} catch (IllegalArgumentException e) {
throw new InterpreterException(e);
} catch (InstantiationException e) {
throw new InterpreterException(e);
} catch (IllegalAccessException e) {
throw new InterpreterException(e);
} catch (InvocationTargetException e) {
throw new InterpreterException(e);
} catch (ClassNotFoundException e) {
throw new InterpreterException(e);
} finally {
Thread.currentThread().setContextClassLoader(oldcl);
}
}
RemoteInterpreterProcess createInterpreterProcess() {
RemoteInterpreterProcess remoteInterpreterProcess = null;
int connectTimeout =
conf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + id;
if (option.isExistingProcess()) {
// TODO(zjffdu) remove the existing process approach seems no one is using this.
// use the existing process
remoteInterpreterProcess = new RemoteInterpreterRunningProcess(
connectTimeout,
remoteInterpreterProcessListener,
appEventListener,
option.getHost(),
option.getPort());
} else {
// create new remote process
remoteInterpreterProcess = new RemoteInterpreterManagedProcess(
interpreterRunner != null ? interpreterRunner.getPath() :
conf.getInterpreterRemoteRunnerPath(), interpreterDir, localRepoPath,
getEnvFromInterpreterProperty(getJavaProperties()), connectTimeout,
remoteInterpreterProcessListener, appEventListener, group);
}
return remoteInterpreterProcess;
}
private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
Map<String, String> env = new HashMap<String, String>();
StringBuilder sparkConfBuilder = new StringBuilder();
for (String key : property.stringPropertyNames()) {
if (RemoteInterpreterUtils.isEnvString(key)) {
env.put(key, property.getProperty(key));
}
if (key.equals("master")) {
sparkConfBuilder.append(" --master " + property.getProperty("master"));
}
if (isSparkConf(key, property.getProperty(key))) {
sparkConfBuilder.append(" --conf " + key + "=" +
toShellFormat(property.getProperty(key)));
}
}
env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());
return env;
}
private String toShellFormat(String value) {
if (value.contains("\'") && value.contains("\"")) {
throw new RuntimeException("Spark property value could not contain both \" and '");
} else if (value.contains("\'")) {
return "\"" + value + "\"";
} else {
return "\'" + value + "\'";
}
}
static boolean isSparkConf(String key, String value) {
return !StringUtils.isEmpty(key) && key.startsWith("spark.") && !StringUtils.isEmpty(value);
}
private List<Interpreter> getOrCreateSession(String user, String noteId) {
InterpreterGroup interpreterGroup = getOrCreateInterpreterGroup(user, noteId);
Preconditions.checkNotNull(interpreterGroup, "No InterpreterGroup existed for user {}, " +
"noteId {}", user, noteId);
String sessionId = getInterpreterSessionId(user, noteId);
return interpreterGroup.getOrCreateSession(user, sessionId);
}
public Interpreter getDefaultInterpreter(String user, String noteId) {
return getOrCreateSession(user, noteId).get(0);
}
public Interpreter getInterpreter(String user, String noteId, String replName) {
Preconditions.checkNotNull(noteId, "noteId should be not null");
Preconditions.checkNotNull(replName, "replName should be not null");
String className = getInterpreterClassFromInterpreterSetting(replName);
if (className == null) {
return null;
}
List<Interpreter> interpreters = getOrCreateSession(user, noteId);
for (Interpreter interpreter : interpreters) {
if (className.equals(interpreter.getClassName())) {
return interpreter;
}
}
return null;
}
private String getInterpreterClassFromInterpreterSetting(String replName) {
Preconditions.checkNotNull(replName, "replName should be not null");
for (InterpreterInfo info : interpreterInfos) {
String infoName = info.getName();
if (null != info.getName() && replName.equals(infoName)) {
return info.getClassName();
}
}
return null;
}
private InterpreterGroup createInterpreterGroup(String groupId) throws InterpreterException {
AngularObjectRegistry angularObjectRegistry;
InterpreterGroup interpreterGroup = new InterpreterGroup(groupId, this);
if (option.isRemote()) {
angularObjectRegistry =
new RemoteAngularObjectRegistry(groupId, angularObjectRegistryListener, interpreterGroup);
} else {
angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener);
// TODO(moon) : create distributed resource pool for local interpreters and set
}
interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
return interpreterGroup;
}
private void loadInterpreterDependencies() {
setStatus(Status.DOWNLOADING_DEPENDENCIES);
setErrorReason(null);
Thread t = new Thread() {
public void run() {
try {
// dependencies to prevent library conflict
File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + getId());
if (localRepoDir.exists()) {
try {
FileUtils.forceDelete(localRepoDir);
} catch (FileNotFoundException e) {
LOGGER.info("A file that does not exist cannot be deleted, nothing to worry", e);
}
}
// load dependencies
List<Dependency> deps = getDependencies();
if (deps != null) {
for (Dependency d : deps) {
File destDir = new File(
conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO));
if (d.getExclusions() != null) {
dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
new File(destDir, id));
} else {
dependencyResolver
.load(d.getGroupArtifactVersion(), new File(destDir, id));
}
}
}
setStatus(Status.READY);
setErrorReason(null);
} catch (Exception e) {
LOGGER.error(String.format("Error while downloading repos for interpreter group : %s," +
" go to interpreter setting page click on edit and save it again to make " +
"this interpreter work properly. : %s",
getGroup(), e.getLocalizedMessage()), e);
setErrorReason(e.getLocalizedMessage());
setStatus(Status.ERROR);
}
}
};
t.start();
}
//TODO(zjffdu) ugly code, should not use JsonObject as parameter. not readable
public void convertPermissionsFromUsersToOwners(JsonObject jsonObject) {
if (jsonObject != null) {
JsonObject option = jsonObject.getAsJsonObject("option");
if (option != null) {
JsonArray users = option.getAsJsonArray("users");
if (users != null) {
if (this.option.getOwners() == null) {
this.option.owners = new LinkedList<>();
}
for (JsonElement user : users) {
this.option.getOwners().add(user.getAsString());
}
}
}
}
}
// For backward compatibility of interpreter.json format after ZEPPELIN-2403
static Map<String, InterpreterProperty> convertInterpreterProperties(Object properties) {
if (properties != null && properties instanceof StringMap) {
Map<String, InterpreterProperty> newProperties = new HashMap<>();
StringMap p = (StringMap) properties;
for (Object o : p.entrySet()) {
Map.Entry entry = (Map.Entry) o;
if (!(entry.getValue() instanceof StringMap)) {
InterpreterProperty newProperty = new InterpreterProperty(
entry.getKey().toString(),
entry.getValue(),
InterpreterPropertyType.STRING.getValue());
newProperties.put(entry.getKey().toString(), newProperty);
} else {
// already converted
return (Map<String, InterpreterProperty>) properties;
}
}
return newProperties;
} else if (properties instanceof Map) {
Map<String, Object> dProperties =
(Map<String, Object>) properties;
Map<String, InterpreterProperty> newProperties = new HashMap<>();
for (String key : dProperties.keySet()) {
Object value = dProperties.get(key);
if (value instanceof InterpreterProperty) {
return (Map<String, InterpreterProperty>) properties;
} else if (value instanceof StringMap) {
StringMap stringMap = (StringMap) value;
InterpreterProperty newProperty = new InterpreterProperty(
key,
stringMap.get("value"),
stringMap.get("type").toString());
newProperties.put(newProperty.getName(), newProperty);
} else if (value instanceof DefaultInterpreterProperty){
DefaultInterpreterProperty dProperty = (DefaultInterpreterProperty) value;
InterpreterProperty property = new InterpreterProperty(
key,
dProperty.getValue(),
dProperty.getType() != null ? dProperty.getType() : "string"
// in case user forget to specify type in interpreter-setting.json
);
newProperties.put(key, property);
} else {
throw new RuntimeException("Can not convert this type of property: " + value.getClass());
}
}
return newProperties;
}
throw new RuntimeException("Can not convert this type: " + properties.getClass());
}
}

886
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java Normal file
View file

@ -0,0 +1,886 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.resource.ResourceSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.repository.Proxy;
import org.sonatype.aether.repository.RemoteRepository;
import java.io.*;
import java.lang.reflect.Type;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
/**
* InterpreterSettingManager is the component which manage all the interpreter settings.
* (load/create/update/remove/get)
* Besides that InterpreterSettingManager also manage the interpreter setting binding.
* TODO(zjffdu) We could move it into another separated component.
*/
public class InterpreterSettingManager {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSettingManager.class);
private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of(
"language", (Object) "text",
"editOnDblClick", false);
private final ZeppelinConfiguration conf;
private final Path interpreterDirPath;
private final Path interpreterSettingPath;
/**
* This is only InterpreterSetting templates with default name and properties
* name --> InterpreterSetting
*/
private final Map<String, InterpreterSetting> interpreterSettingTemplates =
Maps.newConcurrentMap();
/**
* This is used by creating and running Interpreters
* id --> InterpreterSetting
* TODO(zjffdu) change it to name --> InterpreterSetting
*/
private final Map<String, InterpreterSetting> interpreterSettings =
Maps.newConcurrentMap();
/**
* noteId --> list of InterpreterSettingId
*/
private final Map<String, List<String>> interpreterBindings =
Maps.newConcurrentMap();
private final List<RemoteRepository> interpreterRepositories;
private InterpreterOption defaultOption;
private List<String> interpreterGroupOrderList;
private final Gson gson;
private AngularObjectRegistryListener angularObjectRegistryListener;
private RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private ApplicationEventListener appEventListener;
private DependencyResolver dependencyResolver;
public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener
remoteInterpreterProcessListener,
ApplicationEventListener appEventListener)
throws IOException {
this(zeppelinConfiguration, new InterpreterOption(true),
angularObjectRegistryListener,
remoteInterpreterProcessListener,
appEventListener);
}
@VisibleForTesting
public InterpreterSettingManager(ZeppelinConfiguration conf,
InterpreterOption defaultOption,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener
remoteInterpreterProcessListener,
ApplicationEventListener appEventListener) throws IOException {
this.conf = conf;
this.defaultOption = defaultOption;
this.interpreterDirPath = Paths.get(conf.getInterpreterDir());
LOGGER.debug("InterpreterRootPath: {}", interpreterDirPath);
this.interpreterSettingPath = Paths.get(conf.getInterpreterSettingPath());
LOGGER.debug("InterpreterBindingPath: {}", interpreterSettingPath);
this.dependencyResolver = new DependencyResolver(
conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
this.interpreterRepositories = dependencyResolver.getRepos();
this.interpreterGroupOrderList = Arrays.asList(conf.getString(
ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER).split(","));
this.gson = new GsonBuilder().setPrettyPrinting().create();
this.angularObjectRegistryListener = angularObjectRegistryListener;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.appEventListener = appEventListener;
init();
}
/**
* Load interpreter setting from interpreter-setting.json
*/
private void loadFromFile() {
if (!Files.exists(interpreterSettingPath)) {
// nothing to read
LOGGER.warn("Interpreter Setting file {} doesn't exist", interpreterSettingPath);
return;
}
try {
InterpreterInfoSaving infoSaving = InterpreterInfoSaving.loadFromFile(interpreterSettingPath);
//TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving)
for (InterpreterSetting savedInterpreterSetting : infoSaving.interpreterSettings.values()) {
savedInterpreterSetting.setConf(conf);
savedInterpreterSetting.setInterpreterSettingManager(this);
savedInterpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
savedInterpreterSetting.setRemoteInterpreterProcessListener(
remoteInterpreterProcessListener);
savedInterpreterSetting.setAppEventListener(appEventListener);
savedInterpreterSetting.setDependencyResolver(dependencyResolver);
savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties(
savedInterpreterSetting.getProperties()
));
InterpreterSetting interpreterSettingTemplate =
interpreterSettingTemplates.get(savedInterpreterSetting.getGroup());
// InterpreterSettingTemplate is from interpreter-setting.json which represent the latest
// InterpreterSetting, while InterpreterSetting is from interpreter.json which represent
// the user saved interpreter setting
if (interpreterSettingTemplate != null) {
savedInterpreterSetting.setInterpreterDir(interpreterSettingTemplate.getInterpreterDir());
// merge properties from interpreter-setting.json and interpreter.json
Map<String, InterpreterProperty> mergedProperties =
new HashMap<>(InterpreterSetting.convertInterpreterProperties(
interpreterSettingTemplate.getProperties()));
mergedProperties.putAll(InterpreterSetting.convertInterpreterProperties(
savedInterpreterSetting.getProperties()));
savedInterpreterSetting.setProperties(mergedProperties);
// merge InterpreterInfo
savedInterpreterSetting.setInterpreterInfos(
interpreterSettingTemplate.getInterpreterInfos());
} else {
LOGGER.warn("No InterpreterSetting Template found for InterpreterSetting: "
+ savedInterpreterSetting.getGroup());
}
// Overwrite the default InterpreterSetting we registered from InterpreterSetting Templates
// remove it first
for (InterpreterSetting setting : interpreterSettings.values()) {
if (setting.getName().equals(savedInterpreterSetting.getName())) {
interpreterSettings.remove(setting.getId());
}
}
savedInterpreterSetting.postProcessing();
LOGGER.info("Create Interpreter Setting {} from interpreter.json",
savedInterpreterSetting.getName());
interpreterSettings.put(savedInterpreterSetting.getId(), savedInterpreterSetting);
}
interpreterBindings.putAll(infoSaving.interpreterBindings);
if (infoSaving.interpreterRepositories != null) {
for (RemoteRepository repo : infoSaving.interpreterRepositories) {
if (!dependencyResolver.getRepos().contains(repo)) {
this.interpreterRepositories.add(repo);
}
}
}
} catch (IOException e) {
LOGGER.error("Fail to load interpreter setting configuration file: "
+ interpreterSettingPath, e);
}
}
public void saveToFile() throws IOException {
synchronized (interpreterSettings) {
InterpreterInfoSaving info = new InterpreterInfoSaving();
info.interpreterBindings = interpreterBindings;
info.interpreterSettings = interpreterSettings;
info.interpreterRepositories = interpreterRepositories;
info.saveToFile(interpreterSettingPath);
}
}
private void init() throws IOException {
// 1. detect interpreter setting via interpreter-setting.json in each interpreter folder
// 2. detect interpreter setting in interpreter.json that is saved before
String interpreterJson = conf.getInterpreterJson();
ClassLoader cl = Thread.currentThread().getContextClassLoader();
if (Files.exists(interpreterDirPath)) {
for (Path interpreterDir : Files
.newDirectoryStream(interpreterDirPath, new Filter<Path>() {
@Override
public boolean accept(Path entry) throws IOException {
return Files.exists(entry) && Files.isDirectory(entry);
}
})) {
String interpreterDirString = interpreterDir.toString();
/**
* Register interpreter by the following ordering
* 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/
* interpreter-setting.json
* 2. Register it from interpreter-setting.json in classpath
* {ZEPPELIN_HOME}/interpreter/{interpreter_name}
*/
if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) {
if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) {
LOGGER.warn("No interpreter-setting.json found in " + interpreterDirPath);
}
}
}
} else {
LOGGER.warn("InterpreterDir {} doesn't exist", interpreterDirPath);
}
loadFromFile();
saveToFile();
}
private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir,
String interpreterJson) throws IOException {
URL[] urls = recursiveBuildLibList(new File(interpreterDir));
ClassLoader tempClassLoader = new URLClassLoader(urls, cl);
URL url = tempClassLoader.getResource(interpreterJson);
if (url == null) {
return false;
}
LOGGER.debug("Reading interpreter-setting.json from {} as Resource", url);
List<RegisteredInterpreter> registeredInterpreterList =
getInterpreterListFromJson(url.openStream());
registerInterpreterSetting(registeredInterpreterList, interpreterDir);
return true;
}
private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson)
throws IOException {
Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson);
if (Files.exists(interpreterJsonPath)) {
LOGGER.debug("Reading interpreter-setting.json from file {}", interpreterJsonPath);
List<RegisteredInterpreter> registeredInterpreterList =
getInterpreterListFromJson(new FileInputStream(interpreterJsonPath.toFile()));
registerInterpreterSetting(registeredInterpreterList, interpreterDir);
return true;
}
return false;
}
private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) {
Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {
}.getType();
return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType);
}
private void registerInterpreterSetting(List<RegisteredInterpreter> registeredInterpreters,
String interpreterDir) throws IOException {
Map<String, DefaultInterpreterProperty> properties = new HashMap<>();
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
InterpreterOption option = defaultOption;
String group = null;
InterpreterRunner runner = null;
for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) {
//TODO(zjffdu) merge RegisteredInterpreter & InterpreterInfo
InterpreterInfo interpreterInfo =
new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
group = registeredInterpreter.getGroup();
runner = registeredInterpreter.getRunner();
// use defaultOption if it is not specified in interpreter-setting.json
if (registeredInterpreter.getOption() != null) {
option = registeredInterpreter.getOption();
}
properties.putAll(registeredInterpreter.getProperties());
interpreterInfos.add(interpreterInfo);
}
InterpreterSetting interpreterSettingTemplate = new InterpreterSetting.Builder()
.setGroup(group)
.setName(group)
.setInterpreterInfos(interpreterInfos)
.setProperties(properties)
.setDependencies(new ArrayList<Dependency>())
.setOption(option)
.setRunner(runner)
.setInterpreterDir(interpreterDir)
.setRunner(runner)
.setConf(conf)
.setIntepreterSettingManager(this)
.create();
LOGGER.info("Register InterpreterSettingTemplate & InterpreterSetting: {}",
interpreterSettingTemplate.getName());
interpreterSettingTemplates.put(interpreterSettingTemplate.getName(),
interpreterSettingTemplate);
InterpreterSetting interpreterSetting = new InterpreterSetting(interpreterSettingTemplate);
interpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
interpreterSetting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
interpreterSetting.setAppEventListener(appEventListener);
interpreterSetting.setDependencyResolver(dependencyResolver);
interpreterSetting.setInterpreterSettingManager(this);
interpreterSetting.postProcessing();
interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
}
@VisibleForTesting
public InterpreterSetting getDefaultInterpreterSetting(String noteId) {
return getInterpreterSettings(noteId).get(0);
}
public List<InterpreterSetting> getInterpreterSettings(String noteId) {
List<InterpreterSetting> settings = new ArrayList<>();
synchronized (interpreterSettings) {
List<String> interpreterSettingIds = interpreterBindings.get(noteId);
if (interpreterSettingIds != null) {
for (String settingId : interpreterSettingIds) {
if (interpreterSettings.containsKey(settingId)) {
settings.add(interpreterSettings.get(settingId));
} else {
LOGGER.warn("InterpreterSetting {} has been removed, but note {} still bind to it.",
settingId, noteId);
}
}
}
}
return settings;
}
public InterpreterGroup getInterpreterGroupById(String groupId) {
for (InterpreterSetting setting : interpreterSettings.values()) {
InterpreterGroup interpreterGroup = setting.getInterpreterGroup(groupId);
if (interpreterGroup != null) {
return interpreterGroup;
}
}
return null;
}
//TODO(zjffdu) logic here is a little ugly
public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId,
String replName) {
Map<String, Object> editor = DEFAULT_EDITOR;
String group = StringUtils.EMPTY;
try {
String defaultSettingName = getDefaultInterpreterSetting(noteId).getName();
List<InterpreterSetting> intpSettings = getInterpreterSettings(noteId);
for (InterpreterSetting intpSetting : intpSettings) {
String[] replNameSplit = replName.split("\\.");
if (replNameSplit.length == 2) {
group = replNameSplit[0];
}
// when replName is 'name' of interpreter
if (defaultSettingName.equals(intpSetting.getName())) {
editor = intpSetting.getEditorFromSettingByClassName(interpreter.getClassName());
}
// when replName is 'alias name' of interpreter or 'group' of interpreter
if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) {
editor = intpSetting.getEditorFromSettingByClassName(interpreter.getClassName());
break;
}
}
} catch (NullPointerException e) {
// Use `debug` level because this log occurs frequently
LOGGER.debug("Couldn't get interpreter editor setting");
}
return editor;
}
public List<InterpreterGroup> getAllInterpreterGroup() {
List<InterpreterGroup> interpreterGroups = new ArrayList<>();
for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
interpreterGroups.addAll(interpreterSetting.getAllInterpreterGroups());
}
return interpreterGroups;
}
//TODO(zjffdu) move Resource related api to ResourceManager
public ResourceSet getAllResources() {
return getAllResourcesExcept(null);
}
private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : getAllInterpreterGroup()) {
if (interpreterGroupExcludsion != null &&
intpGroup.getId().equals(interpreterGroupExcludsion)) {
continue;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
} else if (remoteInterpreterProcess.isRunning()) {
List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
@Override
public List<String> call(RemoteInterpreterService.Client client) throws Exception {
return client.resourcePoolGetAll();
}
});
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
}
}
return resourceSet;
}
public void removeResourcesBelongsToParagraph(String noteId, String paragraphId) {
for (InterpreterGroup intpGroup : getAllInterpreterGroup()) {
ResourceSet resourceSet = new ResourceSet();
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
if (noteId != null) {
resourceSet = resourceSet.filterByNoteId(noteId);
}
if (paragraphId != null) {
resourceSet = resourceSet.filterByParagraphId(paragraphId);
}
for (Resource r : resourceSet) {
localPool.remove(
r.getResourceId().getNoteId(),
r.getResourceId().getParagraphId(),
r.getResourceId().getName());
}
} else if (remoteInterpreterProcess.isRunning()) {
List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
@Override
public List<String> call(RemoteInterpreterService.Client client) throws Exception {
return client.resourcePoolGetAll();
}
});
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
if (noteId != null) {
resourceSet = resourceSet.filterByNoteId(noteId);
}
if (paragraphId != null) {
resourceSet = resourceSet.filterByParagraphId(paragraphId);
}
for (final Resource r : resourceSet) {
remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(RemoteInterpreterService.Client client) throws Exception {
client.resourceRemove(
r.getResourceId().getNoteId(),
r.getResourceId().getParagraphId(),
r.getResourceId().getName());
return null;
}
});
}
}
}
}
public void removeResourcesBelongsToNote(String noteId) {
removeResourcesBelongsToParagraph(noteId, null);
}
/**
* Overwrite dependency jar under local-repo/{interpreterId}
* if jar file in original path is changed
*/
private void copyDependenciesFromLocalPath(final InterpreterSetting setting) {
setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
synchronized (interpreterSettings) {
final Thread t = new Thread() {
public void run() {
try {
List<Dependency> deps = setting.getDependencies();
if (deps != null) {
for (Dependency d : deps) {
File destDir = new File(
conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
int numSplits = d.getGroupArtifactVersion().split(":").length;
if (!(numSplits >= 3 && numSplits <= 6)) {
dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
new File(destDir, setting.getId()));
}
}
}
setting.setStatus(InterpreterSetting.Status.READY);
} catch (Exception e) {
LOGGER.error(String.format("Error while copying deps for interpreter group : %s," +
" go to interpreter setting page click on edit and save it again to make " +
"this interpreter work properly.",
setting.getGroup()), e);
setting.setErrorReason(e.getLocalizedMessage());
setting.setStatus(InterpreterSetting.Status.ERROR);
} finally {
}
}
};
t.start();
}
}
/**
* Return ordered interpreter setting list.
* The list does not contain more than one setting from the same interpreter class.
* Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name
*/
public List<String> getInterpreterSettingIds() {
List<String> settingIdList = new ArrayList<>();
for (InterpreterSetting interpreterSetting : get()) {
settingIdList.add(interpreterSetting.getId());
}
return settingIdList;
}
public InterpreterSetting createNewSetting(String name, String group,
List<Dependency> dependencies, InterpreterOption option, Map<String, InterpreterProperty> p)
throws IOException {
if (name.indexOf(".") >= 0) {
throw new IOException("'.' is invalid for InterpreterSetting name.");
}
// check if name is existed
for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
if (interpreterSetting.getName().equals(name)) {
throw new IOException("Interpreter " + name + " already existed");
}
}
InterpreterSetting setting = new InterpreterSetting(interpreterSettingTemplates.get(group));
setting.setName(name);
setting.setGroup(group);
//TODO(zjffdu) Should use setDependencies
setting.appendDependencies(dependencies);
setting.setInterpreterOption(option);
setting.setProperties(p);
setting.setAppEventListener(appEventListener);
setting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
setting.setDependencyResolver(dependencyResolver);
setting.setAngularObjectRegistryListener(angularObjectRegistryListener);
setting.setInterpreterSettingManager(this);
setting.postProcessing();
interpreterSettings.put(setting.getId(), setting);
saveToFile();
return setting;
}
@VisibleForTesting
public void addInterpreterSetting(InterpreterSetting interpreterSetting) {
interpreterSettingTemplates.put(interpreterSetting.getName(), interpreterSetting);
interpreterSetting.setAppEventListener(appEventListener);
interpreterSetting.setDependencyResolver(dependencyResolver);
interpreterSetting.setAngularObjectRegistryListener(angularObjectRegistryListener);
interpreterSetting.setRemoteInterpreterProcessListener(remoteInterpreterProcessListener);
interpreterSetting.setInterpreterSettingManager(this);
interpreterSettings.put(interpreterSetting.getId(), interpreterSetting);
}
/**
* map interpreter ids into noteId
*
* @param user user name
* @param noteId note id
* @param settingIdList InterpreterSetting id list
*/
public void setInterpreterBinding(String user, String noteId, List<String> settingIdList)
throws IOException {
List<String> unBindedSettingIdList = new LinkedList<>();
synchronized (interpreterSettings) {
List<String> oldSettingIdList = interpreterBindings.get(noteId);
if (oldSettingIdList != null) {
for (String oldSettingId : oldSettingIdList) {
if (!settingIdList.contains(oldSettingId)) {
unBindedSettingIdList.add(oldSettingId);
}
}
}
interpreterBindings.put(noteId, settingIdList);
saveToFile();
for (String settingId : unBindedSettingIdList) {
InterpreterSetting interpreterSetting = interpreterSettings.get(settingId);
//TODO(zjffdu) Add test for this scenario
//only close Interpreters when it is note scoped
if (interpreterSetting.getOption().perNoteIsolated() ||
interpreterSetting.getOption().perNoteScoped()) {
interpreterSetting.closeInterpreters(user, noteId);
}
}
}
}
public List<String> getInterpreterBinding(String noteId) {
return interpreterBindings.get(noteId);
}
@VisibleForTesting
public void closeNote(String user, String noteId) {
// close interpreters in this note session
LOGGER.info("Close Note: {}", noteId);
List<InterpreterSetting> settings = getInterpreterSettings(noteId);
for (InterpreterSetting setting : settings) {
setting.closeInterpreters(user, noteId);
}
}
public Map<String, InterpreterSetting> getInterpreterSettingTemplates() {
return interpreterSettingTemplates;
}
private URL[] recursiveBuildLibList(File path) throws MalformedURLException {
URL[] urls = new URL[0];
if (path == null || !path.exists()) {
return urls;
} else if (path.getName().startsWith(".")) {
return urls;
} else if (path.isDirectory()) {
File[] files = path.listFiles();
if (files != null) {
for (File f : files) {
urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f));
}
}
return urls;
} else {
return new URL[]{path.toURI().toURL()};
}
}
public List<RemoteRepository> getRepositories() {
return this.interpreterRepositories;
}
public void addRepository(String id, String url, boolean snapshot, Authentication auth,
Proxy proxy) throws IOException {
dependencyResolver.addRepo(id, url, snapshot, auth, proxy);
saveToFile();
}
public void removeRepository(String id) throws IOException {
dependencyResolver.delRepo(id);
saveToFile();
}
public void removeNoteInterpreterSettingBinding(String user, String noteId) throws IOException {
setInterpreterBinding(user, noteId, new ArrayList<String>());
interpreterBindings.remove(noteId);
}
/**
* Change interpreter property and restart
*/
public void setPropertyAndRestart(String id, InterpreterOption option,
Map<String, InterpreterProperty> properties,
List<Dependency> dependencies) throws IOException {
synchronized (interpreterSettings) {
InterpreterSetting intpSetting = interpreterSettings.get(id);
if (intpSetting != null) {
try {
intpSetting.close();
intpSetting.setOption(option);
intpSetting.setProperties(properties);
intpSetting.setDependencies(dependencies);
intpSetting.postProcessing();
saveToFile();
} catch (Exception e) {
loadFromFile();
throw e;
}
} else {
throw new InterpreterException("Interpreter setting id " + id + " not found");
}
}
}
// restart in note page
public void restart(String settingId, String noteId, String user) {
InterpreterSetting intpSetting = interpreterSettings.get(settingId);
Preconditions.checkNotNull(intpSetting);
synchronized (interpreterSettings) {
intpSetting = interpreterSettings.get(settingId);
// Check if dependency in specified path is changed
// If it did, overwrite old dependency jar with new one
if (intpSetting != null) {
//clean up metaInfos
intpSetting.setInfos(null);
copyDependenciesFromLocalPath(intpSetting);
if (user.equals("anonymous")) {
intpSetting.close();
} else {
intpSetting.closeInterpreters(user, noteId);
}
} else {
throw new InterpreterException("Interpreter setting id " + settingId + " not found");
}
}
}
public void restart(String id) {
restart(id, "", "anonymous");
}
public InterpreterSetting get(String id) {
synchronized (interpreterSettings) {
return interpreterSettings.get(id);
}
}
@VisibleForTesting
public InterpreterSetting getByName(String name) {
for (InterpreterSetting interpreterSetting : interpreterSettings.values()) {
if (interpreterSetting.getName().equals(name)) {
return interpreterSetting;
}
}
throw new RuntimeException("No InterpreterSetting: " + name);
}
public void remove(String id) throws IOException {
// 1. close interpreter groups of this interpreter setting
// 2. remove this interpreter setting
// 3. remove this interpreter setting from note binding
// 4. clean local repo directory
LOGGER.info("Remove interpreter setting: " + id);
synchronized (interpreterSettings) {
if (interpreterSettings.containsKey(id)) {
InterpreterSetting intp = interpreterSettings.get(id);
intp.close();
interpreterSettings.remove(id);
for (List<String> settings : interpreterBindings.values()) {
Iterator<String> it = settings.iterator();
while (it.hasNext()) {
String settingId = it.next();
if (settingId.equals(id)) {
it.remove();
}
}
}
saveToFile();
}
}
File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id);
FileUtils.deleteDirectory(localRepoDir);
}
/**
* Get interpreter settings
*/
public List<InterpreterSetting> get() {
synchronized (interpreterSettings) {
List<InterpreterSetting> orderedSettings = new ArrayList<>(interpreterSettings.values());
Collections.sort(orderedSettings, new Comparator<InterpreterSetting>() {
@Override
public int compare(InterpreterSetting o1, InterpreterSetting o2) {
int i = interpreterGroupOrderList.indexOf(o1.getGroup());
int j = interpreterGroupOrderList.indexOf(o2.getGroup());
if (i < 0) {
LOGGER.warn("InterpreterGroup " + o1.getGroup()
+ " is not specified in " + ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName());
// move the unknown interpreter to last
i = Integer.MAX_VALUE;
}
if (j < 0) {
LOGGER.warn("InterpreterGroup " + o2.getGroup()
+ " is not specified in " + ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName());
// move the unknown interpreter to last
j = Integer.MAX_VALUE;
}
if (i < j) {
return -1;
} else if (i > j) {
return 1;
} else {
return 0;
}
}
});
return orderedSettings;
}
}
@VisibleForTesting
public List<String> getSettingIds() {
List<String> settingIds = new ArrayList<>();
for (InterpreterSetting interpreterSetting : get()) {
settingIds.add(interpreterSetting.getId());
}
return settingIds;
}
public void close(String settingId) {
get(settingId).close();
}
public void close() {
List<Thread> closeThreads = new LinkedList<>();
synchronized (interpreterSettings) {
Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
for (final InterpreterSetting intpSetting : intpSettings) {
Thread t = new Thread() {
public void run() {
intpSetting.close();
}
};
t.start();
closeThreads.add(t);
}
}
for (Thread t : closeThreads) {
try {
t.join();
} catch (InterruptedException e) {
LOGGER.error("Can't close interpreterGroup", e);
}
}
}
}

4
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java
View file

@ -17,19 +17,17 @@
package org.apache.zeppelin.interpreter.install;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.ConsoleAppender;
import org.apache.log4j.Logger;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.util.Util;
import org.sonatype.aether.RepositoryException;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

76
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java
View file

@ -17,18 +17,18 @@
package org.apache.zeppelin.interpreter.remote;
import java.util.List;
import com.google.gson.Gson;
import org.apache.thrift.TException;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import java.util.List;
/**
* Proxy for AngularObjectRegistry that exists in remote interpreter process
@ -56,31 +56,29 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* @param noteId
* @return
*/
public AngularObject addAndNotifyRemoteProcess(String name, Object o, String noteId, String
paragraphId) {
Gson gson = new Gson();
public AngularObject addAndNotifyRemoteProcess(final String name,
final Object o,
final String noteId,
final String paragraphId) {
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (!remoteInterpreterProcess.isRunning()) {
return super.add(name, o, noteId, paragraphId, true);
}
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
return super.add(name, o, noteId, paragraphId, true);
} catch (TException e) {
broken = true;
logger.error("Error", e);
} catch (Exception e) {
logger.error("Error", e);
} finally {
if (client != null) {
remoteInterpreterProcess.releaseClient(client, broken);
}
}
return null;
remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
Gson gson = new Gson();
client.angularObjectAdd(name, noteId, paragraphId, gson.toJson(o));
return null;
}
}
);
return super.add(name, o, noteId, paragraphId, true);
}
/**
@ -91,30 +89,24 @@ public class RemoteAngularObjectRegistry extends AngularObjectRegistry {
* @param paragraphId
* @return
*/
public AngularObject removeAndNotifyRemoteProcess(String name, String noteId, String
paragraphId) {
public AngularObject removeAndNotifyRemoteProcess(final String name,
final String noteId,
final String paragraphId) {
RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null || !remoteInterpreterProcess.isRunning()) {
return super.remove(name, noteId, paragraphId);
}
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
client.angularObjectRemove(name, noteId, paragraphId);
return super.remove(name, noteId, paragraphId);
} catch (TException e) {
broken = true;
logger.error("Error", e);
} catch (Exception e) {
logger.error("Error", e);
} finally {
if (client != null) {
remoteInterpreterProcess.releaseClient(client, broken);
remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
client.angularObjectRemove(name, noteId, paragraphId);
return null;
}
}
}
return null;
);
return super.remove(name, noteId, paragraphId);
}
public void removeAllAndNotifyRemoteProcess(String noteId, String paragraphId) {

371
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java Normal file
View file

@ -0,0 +1,371 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.apache.thrift.TException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Proxy for Interpreter instance that runs on separate process
*/
public class RemoteInterpreter extends Interpreter {
private static final Logger LOGGER = LoggerFactory.getLogger(RemoteInterpreter.class);
private static final Gson gson = new Gson();
private String className;
private String sessionId;
private String userName;
private FormType formType;
private RemoteInterpreterProcess interpreterProcess;
private volatile boolean isOpened = false;
private volatile boolean isCreated = false;
/**
* Remote interpreter and manage interpreter process
*/
public RemoteInterpreter(Properties properties,
String sessionId,
String className,
String userName) {
super(properties);
this.sessionId = sessionId;
this.className = className;
this.userName = userName;
}
public boolean isOpened() {
return isOpened;
}
@Override
public String getClassName() {
return className;
}
public String getSessionId() {
return this.sessionId;
}
public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
if (this.interpreterProcess != null) {
return this.interpreterProcess;
}
InterpreterGroup intpGroup = getInterpreterGroup();
this.interpreterProcess = intpGroup.getOrCreateInterpreterProcess();
synchronized (interpreterProcess) {
if (!interpreterProcess.isRunning()) {
interpreterProcess.start(userName, false);
interpreterProcess.getRemoteInterpreterEventPoller()
.setInterpreterProcess(interpreterProcess);
interpreterProcess.getRemoteInterpreterEventPoller().setInterpreterGroup(intpGroup);
interpreterProcess.getRemoteInterpreterEventPoller().start();
}
}
return interpreterProcess;
}
@Override
public void open() {
synchronized (this) {
if (!isOpened) {
// create all the interpreters of the same session first, then Open the internal interpreter
// of this RemoteInterpreter.
// The why we we create all the interpreter of the session is because some interpreter
// depends on other interpreter. e.g. PySparkInterpreter depends on SparkInterpreter.
// also see method Interpreter.getInterpreterInTheSameSessionByClassName
for (Interpreter interpreter : getInterpreterGroup().getOrCreateSession(
userName, sessionId)) {
((RemoteInterpreter) interpreter).internal_create();
}
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
LOGGER.info("Open RemoteInterpreter {}", getClassName());
client.open(sessionId, className);
// Push angular object loaded from JSON file to remote interpreter
synchronized (getInterpreterGroup()) {
if (!getInterpreterGroup().isAngularRegistryPushed()) {
pushAngularObjectRegistryToRemote(client);
getInterpreterGroup().setAngularRegistryPushed(true);
}
}
return null;
}
});
isOpened = true;
}
}
}
private void internal_create() {
synchronized (this) {
if (!isCreated) {
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
LOGGER.info("Create RemoteInterpreter {}", getClassName());
client.createInterpreter(getInterpreterGroup().getId(), sessionId,
className, (Map) property, userName);
return null;
}
});
isCreated = true;
}
}
}
@Override
public void close() {
if (isOpened) {
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
client.close(sessionId, className);
return null;
}
});
} else {
LOGGER.warn("close is called when RemoterInterpreter is not opened for " + className);
}
}
@Override
public InterpreterResult interpret(final String st, final InterpreterContext context) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("st:\n{}", st);
}
final FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
InterpreterContextRunnerPool interpreterContextRunnerPool = interpreterProcess
.getInterpreterContextRunnerPool();
List<InterpreterContextRunner> runners = context.getRunners();
if (runners != null && runners.size() != 0) {
// assume all runners in this InterpreterContext have the same note id
String noteId = runners.get(0).getNoteId();
interpreterContextRunnerPool.clear(noteId);
interpreterContextRunnerPool.addAll(noteId, runners);
}
return interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<InterpreterResult>() {
@Override
public InterpreterResult call(Client client) throws Exception {
RemoteInterpreterResult remoteResult = client.interpret(
sessionId, className, st, convert(context));
Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
}.getType());
context.getConfig().clear();
context.getConfig().putAll(remoteConfig);
GUI currentGUI = context.getGui();
if (form == FormType.NATIVE) {
GUI remoteGui = GUI.fromJson(remoteResult.getGui());
currentGUI.clear();
currentGUI.setParams(remoteGui.getParams());
currentGUI.setForms(remoteGui.getForms());
} else if (form == FormType.SIMPLE) {
final Map<String, Input> currentForms = currentGUI.getForms();
final Map<String, Object> currentParams = currentGUI.getParams();
final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
final Map<String, Input> remoteForms = remoteGUI.getForms();
final Map<String, Object> remoteParams = remoteGUI.getParams();
currentForms.putAll(remoteForms);
currentParams.putAll(remoteParams);
}
InterpreterResult result = convert(remoteResult);
return result;
}
}
);
}
@Override
public void cancel(final InterpreterContext context) {
if (!isOpened) {
LOGGER.warn("Cancel is called when RemoterInterpreter is not opened for " + className);
return;
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
client.cancel(sessionId, className, convert(context));
return null;
}
});
}
@Override
public FormType getFormType() {
if (formType != null) {
return formType;
}
// it is possible to call getFormType before it is opened
synchronized (this) {
if (!isOpened) {
open();
}
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
FormType type = interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<FormType>() {
@Override
public FormType call(Client client) throws Exception {
formType = FormType.valueOf(client.getFormType(sessionId, className));
return formType;
}
});
return type;
}
@Override
public int getProgress(final InterpreterContext context) {
if (!isOpened) {
LOGGER.warn("getProgress is called when RemoterInterpreter is not opened for " + className);
return 0;
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
return interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Integer>() {
@Override
public Integer call(Client client) throws Exception {
return client.getProgress(sessionId, className, convert(context));
}
});
}
@Override
public List<InterpreterCompletion> completion(final String buf, final int cursor,
final InterpreterContext interpreterContext) {
if (!isOpened) {
LOGGER.warn("completion is called when RemoterInterpreter is not opened for " + className);
return new ArrayList<>();
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
return interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<List<InterpreterCompletion>>() {
@Override
public List<InterpreterCompletion> call(Client client) throws Exception {
return client.completion(sessionId, className, buf, cursor,
convert(interpreterContext));
}
});
}
public String getStatus(final String jobId) {
if (!isOpened) {
LOGGER.warn("getStatus is called when RemoteInterpreter is not opened for " + className);
return Job.Status.UNKNOWN.name();
}
RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess();
return interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<String>() {
@Override
public String call(Client client) throws Exception {
return client.getStatus(sessionId, jobId);
}
});
}
//TODO(zjffdu) Share the Scheduler in the same session or in the same InterpreterGroup ?
@Override
public Scheduler getScheduler() {
int maxConcurrency = Integer.parseInt(
property.getProperty("zeppelin.interpreter.max.poolsize",
ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE.getIntValue() + ""));
return SchedulerFactory.singleton().createOrGetRemoteScheduler(
RemoteInterpreter.class.getName() + "-" + sessionId,
sessionId, this, maxConcurrency);
}
private RemoteInterpreterContext convert(InterpreterContext ic) {
return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(),
ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getAuthenticationInfo()),
gson.toJson(ic.getConfig()), gson.toJson(ic.getGui()), gson.toJson(ic.getRunners()));
}
private InterpreterResult convert(RemoteInterpreterResult result) {
InterpreterResult r = new InterpreterResult(
InterpreterResult.Code.valueOf(result.getCode()));
for (RemoteInterpreterResultMessage m : result.getMsg()) {
r.add(InterpreterResult.Type.valueOf(m.getType()), m.getData());
}
return r;
}
/**
* Push local angular object registry to
* remote interpreter. This method should be
* call ONLY once when the first Interpreter is created
*/
private void pushAngularObjectRegistryToRemote(Client client) throws TException {
final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup()
.getAngularObjectRegistry();
if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) {
final Map<String, Map<String, AngularObject>> registry = angularObjectRegistry
.getRegistry();
LOGGER.info("Push local angular object registry from ZeppelinServer to" +
" remote interpreter group {}", this.getInterpreterGroup().getId());
final java.lang.reflect.Type registryType = new TypeToken<Map<String,
Map<String, AngularObject>>>() {
}.getType();
client.angularRegistryPush(gson.toJson(registry, registryType));
}
}
@Override
public String toString() {
return "RemoteInterpreter_" + className + "_" + sessionId;
}
}

281
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -29,6 +29,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner;
import org.apache.zeppelin.resource.Resource;
@ -38,6 +39,7 @@ import org.apache.zeppelin.resource.ResourceSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
@ -84,7 +86,6 @@ public class RemoteInterpreterEventPoller extends Thread {
@Override
public void run() {
Client client = null;
AppendOutputRunner runner = new AppendOutputRunner(listener);
ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay(
runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS);
@ -100,26 +101,14 @@ public class RemoteInterpreterEventPoller extends Thread {
continue;
}
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
logger.error("Can't get RemoteInterpreterEvent", e1);
waitQuietly();
continue;
}
RemoteInterpreterEvent event = null;
boolean broken = false;
try {
event = client.getEvent();
} catch (TException e) {
broken = true;
logger.error("Can't get RemoteInterpreterEvent", e);
waitQuietly();
continue;
} finally {
interpreterProcess.releaseClient(client, broken);
}
RemoteInterpreterEvent event = interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<RemoteInterpreterEvent>() {
@Override
public RemoteInterpreterEvent call(Client client) throws Exception {
return client.getEvent();
}
}
);
AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry();
@ -286,10 +275,7 @@ public class RemoteInterpreterEventPoller extends Thread {
boolean broken = false;
final Gson gson = new Gson();
final String eventOwnerKey = reqResourceBody.getOwnerKey();
Client interpreterServerMain = null;
try {
interpreterServerMain = interpreterProcess.getClient();
final Client eventClient = interpreterServerMain;
if (resourceType == RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS) {
final List<ZeppelinServerResourceParagraphRunner> remoteRunners = new LinkedList<>();
@ -308,7 +294,6 @@ public class RemoteInterpreterEventPoller extends Thread {
@Override
public void onFinished(Object resultObject) {
boolean clientBroken = false;
if (resultObject != null && resultObject instanceof List) {
List<InterpreterContextRunner> runnerList =
(List<InterpreterContextRunner>) resultObject;
@ -324,15 +309,15 @@ public class RemoteInterpreterEventPoller extends Thread {
resResource.setResourceType(RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS);
resResource.setData(remoteRunners);
try {
eventClient.onReceivedZeppelinResource(resResource.toJson());
} catch (Exception e) {
clientBroken = true;
logger.error("Can't get RemoteInterpreterEvent", e);
waitQuietly();
} finally {
interpreterProcess.releaseClient(eventClient, clientBroken);
}
interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
client.onReceivedZeppelinResource(resResource.toJson());
return null;
}
}
);
}
}
@ -346,39 +331,32 @@ public class RemoteInterpreterEventPoller extends Thread {
reqRunnerContext.getNoteId(), reqRunnerContext.getParagraphId(), callBackEvent);
}
} catch (Exception e) {
broken = true;
logger.error("Can't get RemoteInterpreterEvent", e);
waitQuietly();
} finally {
interpreterProcess.releaseClient(interpreterServerMain, broken);
}
}
private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) {
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
List<String> resourceList = new LinkedList<>();
Gson gson = new Gson();
for (Resource r : resourceSet) {
resourceList.add(gson.toJson(r));
}
client.resourcePoolResponseGetAll(resourceList);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
}
private void sendResourcePoolResponseGetAll(final ResourceSet resourceSet) {
interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
List<String> resourceList = new LinkedList<>();
for (Resource r : resourceSet) {
resourceList.add(r.toJson());
}
client.resourcePoolResponseGetAll(resourceList);
return null;
}
}
);
}
private ResourceSet getAllResourcePoolExcept() {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
for (InterpreterGroup intpGroup : interpreterGroup.getInterpreterSetting()
.getInterpreterSettingManager().getAllInterpreterGroup()) {
if (intpGroup.getId().equals(interpreterGroup.getId())) {
continue;
}
@ -390,115 +368,94 @@ public class RemoteInterpreterEventPoller extends Thread {
resourceSet.addAll(localPool.getAll());
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
List<String> resourceList = client.resourcePoolGetAll();
Gson gson = new Gson();
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
@Override
public List<String> call(Client client) throws Exception {
return client.resourcePoolGetAll();
}
}
);
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
}
}
return resourceSet;
}
private void sendResourceResponseGet(ResourceId resourceId, Object o) {
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
Gson gson = new Gson();
String rid = gson.toJson(resourceId);
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseGet(rid, obj);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
}
private void sendResourceResponseGet(final ResourceId resourceId, final Object o) {
interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
String rid = resourceId.toJson();
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseGet(rid, obj);
return null;
}
}
);
}
private Object getResource(ResourceId resourceId) {
InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
resourceId.getResourcePoolId());
private Object getResource(final ResourceId resourceId) {
InterpreterGroup intpGroup = interpreterGroup.getInterpreterSetting()
.getInterpreterSettingManager()
.getInterpreterGroupById(resourceId.getResourcePoolId());
if (intpGroup == null) {
return null;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
return localPool.get(resourceId.getName());
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
ByteBuffer res = client.resourceGet(
resourceId.getNoteId(),
resourceId.getParagraphId(),
resourceId.getName());
Object o = Resource.deserializeObject(res);
return o;
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
ByteBuffer buffer = remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<ByteBuffer>() {
@Override
public ByteBuffer call(Client client) throws Exception {
return client.resourceGet(
resourceId.getNoteId(),
resourceId.getParagraphId(),
resourceId.getName());
}
}
}
);
try {
Object o = Resource.deserializeObject(buffer);
return o;
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return null;
}
public void sendInvokeMethodResult(InvokeResourceMethodEventMessage message, Object o) {
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
Gson gson = new Gson();
String invokeMessage = gson.toJson(message);
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseInvokeMethod(invokeMessage, obj);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
}
public void sendInvokeMethodResult(final InvokeResourceMethodEventMessage message,
final Object o) {
interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<Void>() {
@Override
public Void call(Client client) throws Exception {
String invokeMessage = message.toJson();
ByteBuffer obj;
if (o == null) {
obj = ByteBuffer.allocate(0);
} else {
obj = Resource.serializeObject(o);
}
client.resourceResponseInvokeMethod(invokeMessage, obj);
return null;
}
}
);
}
private Object invokeResourceMethod(InvokeResourceMethodEventMessage message) {
ResourceId resourceId = message.resourceId;
InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId(
resourceId.getResourcePoolId());
private Object invokeResourceMethod(final InvokeResourceMethodEventMessage message) {
final ResourceId resourceId = message.resourceId;
InterpreterGroup intpGroup = interpreterGroup.getInterpreterSetting()
.getInterpreterSettingManager().getInterpreterGroupById(resourceId.getResourcePoolId());
if (intpGroup == null) {
return null;
}
@ -529,25 +486,25 @@ public class RemoteInterpreterEventPoller extends Thread {
return null;
}
} else if (interpreterProcess.isRunning()) {
Client client = null;
boolean broken = false;
ByteBuffer res = interpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<ByteBuffer>() {
@Override
public ByteBuffer call(Client client) throws Exception {
return client.resourceInvokeMethod(
resourceId.getNoteId(),
resourceId.getParagraphId(),
resourceId.getName(),
message.toJson());
}
}
);
try {
client = remoteInterpreterProcess.getClient();
ByteBuffer res = client.resourceInvokeMethod(
resourceId.getNoteId(),
resourceId.getParagraphId(),
resourceId.getName(),
gson.toJson(message));
Object o = Resource.deserializeObject(res);
return o;
return Resource.deserializeObject(res);
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
return null;
}
return null;
}

13
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
View file

@ -21,6 +21,7 @@ import org.apache.commons.exec.*;
import org.apache.commons.exec.environment.EnvironmentUtils;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -97,6 +98,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
// start server process
try {
port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
logger.info("Choose port {} for RemoteInterpreterProcess", port);
} catch (IOException e1) {
throw new InterpreterException(e1);
}
@ -172,6 +174,17 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
public void stop() {
if (isRunning()) {
logger.info("kill interpreter process");
try {
callRemoteFunction(new RemoteFunction<Void>() {
@Override
public Void call(RemoteInterpreterService.Client client) throws Exception {
client.shutdown();
return null;
}
});
} catch (Exception e) {
logger.warn("ignore the exception when shutting down");
}
watchdog.destroyProcess();
}

154
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
View file

@ -20,10 +20,13 @@ import com.google.gson.Gson;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.thrift.TException;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
/**
@ -32,9 +35,6 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class RemoteInterpreterProcess {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class);
// number of sessions that are attached to this process
private final AtomicInteger referenceCount;
private GenericObjectPool<Client> clientPool;
private final RemoteInterpreterEventPoller remoteInterpreterEventPoller;
private final InterpreterContextRunnerPool interpreterContextRunnerPool;
@ -46,16 +46,20 @@ public abstract class RemoteInterpreterProcess {
ApplicationEventListener appListener) {
this(new RemoteInterpreterEventPoller(listener, appListener),
connectTimeout);
this.remoteInterpreterEventPoller.setInterpreterProcess(this);
}
RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller,
int connectTimeout) {
this.interpreterContextRunnerPool = new InterpreterContextRunnerPool();
referenceCount = new AtomicInteger(0);
this.remoteInterpreterEventPoller = remoteInterpreterEventPoller;
this.connectTimeout = connectTimeout;
}
public RemoteInterpreterEventPoller getRemoteInterpreterEventPoller() {
return remoteInterpreterEventPoller;
}
public abstract String getHost();
public abstract int getPort();
public abstract void start(String userName, Boolean isUserImpersonate);
@ -66,37 +70,18 @@ public abstract class RemoteInterpreterProcess {
return connectTimeout;
}
public int reference(InterpreterGroup interpreterGroup, String userName,
Boolean isUserImpersonate) {
synchronized (referenceCount) {
if (!isRunning()) {
start(userName, isUserImpersonate);
}
if (clientPool == null) {
clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
clientPool.setTestOnBorrow(true);
remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup);
remoteInterpreterEventPoller.setInterpreterProcess(this);
remoteInterpreterEventPoller.start();
}
return referenceCount.incrementAndGet();
}
}
public Client getClient() throws Exception {
public synchronized Client getClient() throws Exception {
if (clientPool == null || clientPool.isClosed()) {
return null;
clientPool = new GenericObjectPool<>(new ClientFactory(getHost(), getPort()));
}
return clientPool.borrowObject();
}
public void releaseClient(Client client) {
private void releaseClient(Client client) {
releaseClient(client, false);
}
public void releaseClient(Client client, boolean broken) {
private void releaseClient(Client client, boolean broken) {
if (broken) {
releaseBrokenClient(client);
} else {
@ -108,7 +93,7 @@ public abstract class RemoteInterpreterProcess {
}
}
public void releaseBrokenClient(Client client) {
private void releaseBrokenClient(Client client) {
try {
clientPool.invalidateObject(client);
} catch (Exception e) {
@ -116,90 +101,6 @@ public abstract class RemoteInterpreterProcess {
}
}
public int dereference() {
synchronized (referenceCount) {
int r = referenceCount.decrementAndGet();
if (r == 0) {
logger.info("shutdown interpreter process");
remoteInterpreterEventPoller.shutdown();
// first try shutdown
Client client = null;
try {
client = getClient();
client.shutdown();
} catch (Exception e) {
// safely ignore exception while client.shutdown() may terminates remote process
logger.info("Exception in RemoteInterpreterProcess while synchronized dereference, can " +
"safely ignore exception while client.shutdown() may terminates remote process");
logger.debug(e.getMessage(), e);
} finally {
if (client != null) {
// no longer used
releaseBrokenClient(client);
}
}
clientPool.clear();
clientPool.close();
// wait for some time (connectTimeout) and force kill
// remote process server.serve() loop is not always finishing gracefully
long startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < connectTimeout) {
if (this.isRunning()) {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
logger.error("Exception in RemoteInterpreterProcess while synchronized dereference " +
"Thread.sleep", e);
}
} else {
break;
}
}
}
return r;
}
}
public int referenceCount() {
synchronized (referenceCount) {
return referenceCount.get();
}
}
public int getNumActiveClient() {
if (clientPool == null) {
return 0;
} else {
return clientPool.getNumActive();
}
}
public int getNumIdleClient() {
if (clientPool == null) {
return 0;
} else {
return clientPool.getNumIdle();
}
}
public void setMaxPoolSize(int size) {
if (clientPool != null) {
//Size + 2 for progress poller , cancel operation
clientPool.setMaxTotal(size + 2);
}
}
public int getMaxPoolSize() {
if (clientPool != null) {
return clientPool.getMaxTotal();
} else {
return 0;
}
}
/**
* Called when angular object is updated in client side to propagate
* change to the remote process
@ -239,4 +140,33 @@ public abstract class RemoteInterpreterProcess {
public InterpreterContextRunnerPool getInterpreterContextRunnerPool() {
return interpreterContextRunnerPool;
}
public <T> T callRemoteFunction(RemoteFunction<T> func) {
Client client = null;
boolean broken = false;
try {
client = getClient();
if (client != null) {
return func.call(client);
}
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} catch (Exception e1) {
throw new InterpreterException(e1);
} finally {
if (client != null) {
releaseClient(client, broken);
}
}
return null;
}
/**
*
* @param <T>
*/
public interface RemoteFunction<T> {
T call(Client client) throws Exception;
}
}

0
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java
View file

58
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java
View file

@ -106,6 +106,7 @@ public class RemoteInterpreterServer
@Override
public void shutdown() throws TException {
logger.info("Shutting down...");
eventClient.waitForEventQueueBecomesEmpty(DEFAULT_SHUTDOWN_TIMEOUT);
if (interpreterGroup != null) {
interpreterGroup.close();
@ -159,7 +160,7 @@ public class RemoteInterpreterServer
}
@Override
public void createInterpreter(String interpreterGroupId, String sessionKey, String
public void createInterpreter(String interpreterGroupId, String sessionId, String
className, Map<String, String> properties, String userName) throws TException {
if (interpreterGroup == null) {
interpreterGroup = new InterpreterGroup(interpreterGroupId);
@ -190,20 +191,11 @@ public class RemoteInterpreterServer
replClass.getConstructor(new Class[] {Properties.class});
Interpreter repl = constructor.newInstance(p);
repl.setClassloaderUrls(new URL[]{});
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
if (interpreters == null) {
interpreters = new LinkedList<>();
interpreterGroup.put(sessionKey, interpreters);
}
interpreters.add(new LazyOpenInterpreter(repl));
}
logger.info("Instantiate interpreter {}", className);
repl.setInterpreterGroup(interpreterGroup);
repl.setUserName(userName);
interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(repl), sessionId);
} catch (ClassNotFoundException | NoSuchMethodException | SecurityException
| InstantiationException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException e) {
@ -237,13 +229,13 @@ public class RemoteInterpreterServer
}
}
protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
protected Interpreter getInterpreter(String sessionId, String className) throws TException {
if (interpreterGroup == null) {
throw new TException(
new InterpreterException("Interpreter instance " + className + " not created"));
}
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
List<Interpreter> interpreters = interpreterGroup.get(sessionId);
if (interpreters == null) {
throw new TException(
new InterpreterException("Interpreter " + className + " not initialized"));
@ -259,19 +251,20 @@ public class RemoteInterpreterServer
}
@Override
public void open(String noteId, String className) throws TException {
Interpreter intp = getInterpreter(noteId, className);
public void open(String sessionId, String className) throws TException {
logger.info(String.format("Open Interpreter %s for session %s ", className, sessionId));
Interpreter intp = getInterpreter(sessionId, className);
intp.open();
}
@Override
public void close(String sessionKey, String className) throws TException {
public void close(String sessionId, String className) throws TException {
// unload all applications
for (String appId : runningApplications.keySet()) {
RunningApplication appInfo = runningApplications.get(appId);
// see NoteInterpreterLoader.SHARED_SESSION
if (appInfo.noteId.equals(sessionKey) || sessionKey.equals("shared_session")) {
if (appInfo.noteId.equals(sessionId) || sessionId.equals("shared_session")) {
try {
logger.info("Unload App {} ", appInfo.pkg.getName());
appInfo.app.unload();
@ -286,7 +279,7 @@ public class RemoteInterpreterServer
// close interpreters
List<Interpreter> interpreters;
synchronized (interpreterGroup) {
interpreters = interpreterGroup.get(sessionKey);
interpreters = interpreterGroup.get(sessionId);
}
if (interpreters != null) {
Iterator<Interpreter> it = interpreters.iterator();
@ -322,7 +315,6 @@ public class RemoteInterpreterServer
intp,
st,
context);
scheduler.submit(job);
while (!job.isTerminated()) {
@ -559,30 +551,34 @@ public class RemoteInterpreterServer
}
@Override
public int getProgress(String noteId, String className,
public int getProgress(String sessionId, String className,
RemoteInterpreterContext interpreterContext)
throws TException {
Integer manuallyProvidedProgress = progressMap.get(interpreterContext.getParagraphId());
if (manuallyProvidedProgress != null) {
return manuallyProvidedProgress;
} else {
Interpreter intp = getInterpreter(noteId, className);
Interpreter intp = getInterpreter(sessionId, className);
if (intp == null) {
throw new TException("No interpreter {} existed for session {}".format(
className, sessionId));
}
return intp.getProgress(convert(interpreterContext, null));
}
}
@Override
public String getFormType(String noteId, String className) throws TException {
Interpreter intp = getInterpreter(noteId, className);
public String getFormType(String sessionId, String className) throws TException {
Interpreter intp = getInterpreter(sessionId, className);
return intp.getFormType().toString();
}
@Override
public List<InterpreterCompletion> completion(String noteId,
public List<InterpreterCompletion> completion(String sessionId,
String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext)
throws TException {
Interpreter intp = getInterpreter(noteId, className);
Interpreter intp = getInterpreter(sessionId, className);
List completion = intp.completion(buf, cursor, convert(remoteInterpreterContext, null));
return completion;
}
@ -759,16 +755,16 @@ public class RemoteInterpreterServer
}
@Override
public String getStatus(String sessionKey, String jobId)
public String getStatus(String sessionId, String jobId)
throws TException {
if (interpreterGroup == null) {
return "Unknown";
return Status.UNKNOWN.name();
}
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
List<Interpreter> interpreters = interpreterGroup.get(sessionId);
if (interpreters == null) {
return "Unknown";
return Status.UNKNOWN.name();
}
for (Interpreter intp : interpreters) {
@ -785,7 +781,7 @@ public class RemoteInterpreterServer
}
}
}
return "Unknown";
return Status.UNKNOWN.name();
}

138
zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java
View file

@ -1,138 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.resource;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.slf4j.Logger;
import java.util.List;
/**
* Utilities for ResourcePool
*/
public class ResourcePoolUtils {
static Logger logger = org.slf4j.LoggerFactory.getLogger(ResourcePoolUtils.class);
public static ResourceSet getAllResources() {
return getAllResourcesExcept(null);
}
public static ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
if (interpreterGroupExcludsion != null &&
intpGroup.getId().equals(interpreterGroupExcludsion)) {
continue;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
} else if (remoteInterpreterProcess.isRunning()) {
RemoteInterpreterService.Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
if (client == null) {
// remote interpreter may not started yet or terminated.
continue;
}
List<String> resourceList = client.resourcePoolGetAll();
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
}
}
return resourceSet;
}
public static void removeResourcesBelongsToNote(String noteId) {
removeResourcesBelongsToParagraph(noteId, null);
}
public static void removeResourcesBelongsToParagraph(String noteId, String paragraphId) {
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
ResourceSet resourceSet = new ResourceSet();
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
if (noteId != null) {
resourceSet = resourceSet.filterByNoteId(noteId);
}
if (paragraphId != null) {
resourceSet = resourceSet.filterByParagraphId(paragraphId);
}
for (Resource r : resourceSet) {
localPool.remove(
r.getResourceId().getNoteId(),
r.getResourceId().getParagraphId(),
r.getResourceId().getName());
}
} else if (remoteInterpreterProcess.isRunning()) {
RemoteInterpreterService.Client client = null;
boolean broken = false;
try {
client = remoteInterpreterProcess.getClient();
List<String> resourceList = client.resourcePoolGetAll();
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
if (noteId != null) {
resourceSet = resourceSet.filterByNoteId(noteId);
}
if (paragraphId != null) {
resourceSet = resourceSet.filterByParagraphId(paragraphId);
}
for (Resource r : resourceSet) {
client.resourceRemove(
r.getResourceId().getNoteId(),
r.getResourceId().getParagraphId(),
r.getResourceId().getName());
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
broken = true;
} finally {
if (client != null) {
intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken);
}
}
}
}
}
}

11
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
View file

@ -41,6 +41,7 @@ public abstract class Job {
/**
* Job status.
*
* UNKNOWN - Job is not found in remote
* READY - Job is not running, ready to run.
* PENDING - Job is submitted to scheduler. but not running yet
* RUNNING - Job is running.
@ -48,8 +49,8 @@ public abstract class Job {
* ERROR - Job finished run. with error
* ABORT - Job finished by abort
*/
public static enum Status {
READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
public enum Status {
UNKNOWN, READY, PENDING, RUNNING, FINISHED, ERROR, ABORT;
public boolean isReady() {
return this == READY;
@ -70,14 +71,14 @@ public abstract class Job {
Date dateCreated;
Date dateStarted;
Date dateFinished;
Status status;
volatile Status status;
static Logger LOGGER = LoggerFactory.getLogger(Job.class);
transient boolean aborted = false;
private String errorMessage;
private transient Throwable exception;
private volatile String errorMessage;
private transient volatile Throwable exception;
private transient JobListener listener;
private long progressUpdateIntervalMs;

134
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java
View file

@ -17,11 +17,9 @@
package org.apache.zeppelin.scheduler;
import org.apache.thrift.TException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.scheduler.Job.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,6 +32,7 @@ import java.util.concurrent.ExecutorService;
/**
* RemoteScheduler runs in ZeppelinServer and proxies Scheduler running on RemoteInterpreter
*
*/
public class RemoteScheduler implements Scheduler {
Logger logger = LoggerFactory.getLogger(RemoteScheduler.class);
@ -45,17 +44,17 @@ public class RemoteScheduler implements Scheduler {
boolean terminate = false;
private String name;
private int maxConcurrency;
private final String noteId;
private RemoteInterpreterProcess interpreterProcess;
private final String sessionId;
private RemoteInterpreter remoteInterpreter;
public RemoteScheduler(String name, ExecutorService executor, String noteId,
RemoteInterpreterProcess interpreterProcess, SchedulerListener listener,
public RemoteScheduler(String name, ExecutorService executor, String sessionId,
RemoteInterpreter remoteInterpreter, SchedulerListener listener,
int maxConcurrency) {
this.name = name;
this.executor = executor;
this.listener = listener;
this.noteId = noteId;
this.interpreterProcess = interpreterProcess;
this.sessionId = sessionId;
this.remoteInterpreter = remoteInterpreter;
this.maxConcurrency = maxConcurrency;
}
@ -167,14 +166,15 @@ public class RemoteScheduler implements Scheduler {
private long initialPeriodMsec;
private long initialPeriodCheckIntervalMsec;
private long checkIntervalMsec;
private boolean terminate;
private volatile boolean terminate;
private JobListener listener;
private Job job;
Status lastStatus;
volatile Status lastStatus;
public JobStatusPoller(long initialPeriodMsec,
long initialPeriodCheckIntervalMsec, long checkIntervalMsec, Job job,
JobListener listener) {
setName("JobStatusPoller-" + job.getId());
this.initialPeriodMsec = initialPeriodMsec;
this.initialPeriodCheckIntervalMsec = initialPeriodCheckIntervalMsec;
this.checkIntervalMsec = checkIntervalMsec;
@ -209,7 +209,7 @@ public class RemoteScheduler implements Scheduler {
}
Status newStatus = getStatus();
if (newStatus == null) { // unknown
if (newStatus == Status.UNKNOWN) { // unknown
continue;
}
@ -231,7 +231,9 @@ public class RemoteScheduler implements Scheduler {
private Status getLastStatus() {
if (terminate == true) {
if (lastStatus != Status.FINISHED &&
if (job.getErrorMessage() != null) {
return Status.ERROR;
} else if (lastStatus != Status.FINISHED &&
lastStatus != Status.ERROR &&
lastStatus != Status.ABORT) {
return Status.FINISHED;
@ -239,58 +241,35 @@ public class RemoteScheduler implements Scheduler {
return (lastStatus == null) ? Status.FINISHED : lastStatus;
}
} else {
return (lastStatus == null) ? Status.FINISHED : lastStatus;
return (lastStatus == null) ? Status.UNKNOWN : lastStatus;
}
}
public synchronized Job.Status getStatus() {
if (interpreterProcess.referenceCount() <= 0) {
if (!remoteInterpreter.isOpened()) {
return getLastStatus();
}
Client client;
try {
client = interpreterProcess.getClient();
} catch (Exception e) {
logger.error("Can't get status information", e);
lastStatus = Status.ERROR;
return Status.ERROR;
}
boolean broken = false;
try {
String statusStr = client.getStatus(noteId, job.getId());
if ("Unknown".equals(statusStr)) {
// not found this job in the remote schedulers.
// maybe not submitted, maybe already finished
//Status status = getLastStatus();
listener.afterStatusChange(job, null, null);
return job.getStatus();
}
Status status = Status.valueOf(statusStr);
lastStatus = status;
listener.afterStatusChange(job, null, status);
return status;
} catch (TException e) {
broken = true;
logger.error("Can't get status information", e);
lastStatus = Status.ERROR;
return Status.ERROR;
} catch (Exception e) {
logger.error("Unknown status", e);
lastStatus = Status.ERROR;
return Status.ERROR;
} finally {
interpreterProcess.releaseClient(client, broken);
Status status = Status.valueOf(remoteInterpreter.getStatus(job.getId()));
if (status == Status.UNKNOWN) {
// not found this job in the remote schedulers.
// maybe not submitted, maybe already finished
//Status status = getLastStatus();
listener.afterStatusChange(job, null, null);
return job.getStatus();
}
lastStatus = status;
listener.afterStatusChange(job, null, status);
return status;
}
}
//TODO(zjffdu) need to refactor the schdule module which is too complicated
private class JobRunner implements Runnable, JobListener {
private final Logger logger = LoggerFactory.getLogger(JobRunner.class);
private Scheduler scheduler;
private Job job;
private boolean jobExecuted;
boolean jobSubmittedRemotely;
private volatile boolean jobExecuted;
volatile boolean jobSubmittedRemotely;
public JobRunner(Scheduler scheduler, Job job) {
this.scheduler = scheduler;
@ -338,20 +317,22 @@ public class RemoteScheduler implements Scheduler {
}
// set job status based on result.
Status lastStatus = jobStatusPoller.getStatus();
Object jobResult = job.getReturn();
if (jobResult != null && jobResult instanceof InterpreterResult) {
if (((InterpreterResult) jobResult).code() == Code.ERROR) {
lastStatus = Status.ERROR;
}
}
if (job.getException() != null) {
lastStatus = Status.ERROR;
if (job.isAborted()) {
job.setStatus(Status.ABORT);
} else if (job.getException() != null) {
// logger.info("Job ABORT, " + job.getId());
job.setStatus(Status.ERROR);
} else if (jobResult != null && jobResult instanceof InterpreterResult
&& ((InterpreterResult) jobResult).code() == Code.ERROR) {
// logger.info("Job Error, " + job.getId());
job.setStatus(Status.ERROR);
} else {
// logger.info("Job Finished, " + job.getId());
job.setStatus(Status.FINISHED);
}
synchronized (queue) {
job.setStatus(lastStatus);
if (listener != null) {
listener.jobFinished(scheduler, job);
}
@ -374,25 +355,6 @@ public class RemoteScheduler implements Scheduler {
@Override
public void afterStatusChange(Job job, Status before, Status after) {
if (after == null) { // unknown. maybe before sumitted remotely, maybe already finished.
if (jobExecuted) {
jobSubmittedRemotely = true;
Object jobResult = job.getReturn();
if (job.isAborted()) {
job.setStatus(Status.ABORT);
} else if (job.getException() != null) {
job.setStatus(Status.ERROR);
} else if (jobResult != null && jobResult instanceof InterpreterResult
&& ((InterpreterResult) jobResult).code() == Code.ERROR) {
job.setStatus(Status.ERROR);
} else {
job.setStatus(Status.FINISHED);
}
}
return;
}
// Update remoteStatus
if (jobExecuted == false) {
if (after == Status.FINISHED || after == Status.ABORT
@ -402,14 +364,18 @@ public class RemoteScheduler implements Scheduler {
return;
} else if (after == Status.RUNNING) {
jobSubmittedRemotely = true;
job.setStatus(Status.RUNNING);
// logger.info("Job RUNNING, " + job.getId());
}
} else {
jobSubmittedRemotely = true;
}
// status polled by status poller
if (job.getStatus() != after) {
job.setStatus(after);
// only set status when it is RUNNING
// We would set other status based on the interpret result
if (after == Status.RUNNING) {
// logger.info("Job RUNNING, " + job.getId());
job.setStatus(Status.RUNNING);
}
}
}

49
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/SchedulerFactory.java
View file

@ -24,17 +24,18 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* TODO(moon) : add description.
* Factory class for creating schedulers
*
*/
public class SchedulerFactory implements SchedulerListener {
private static final Logger logger = LoggerFactory.getLogger(SchedulerFactory.class);
ExecutorService executor;
Map<String, Scheduler> schedulers = new LinkedHashMap<>();
private ExecutorService executor;
private Map<String, Scheduler> schedulers = new LinkedHashMap<>();
private static SchedulerFactory singleton;
private static Long singletonLock = new Long(0);
@ -54,17 +55,17 @@ public class SchedulerFactory implements SchedulerListener {
return singleton;
}
public SchedulerFactory() throws Exception {
executor = ExecutorFactory.singleton().createOrGet("schedulerFactory", 100);
SchedulerFactory() throws Exception {
executor = ExecutorFactory.singleton().createOrGet("SchedulerFactory", 100);
}
public void destroy() {
ExecutorFactory.singleton().shutdown("schedulerFactory");
ExecutorFactory.singleton().shutdown("SchedulerFactory");
}
public Scheduler createOrGetFIFOScheduler(String name) {
synchronized (schedulers) {
if (schedulers.containsKey(name) == false) {
if (!schedulers.containsKey(name)) {
Scheduler s = new FIFOScheduler(name, executor, this);
schedulers.put(name, s);
executor.execute(s);
@ -75,7 +76,7 @@ public class SchedulerFactory implements SchedulerListener {
public Scheduler createOrGetParallelScheduler(String name, int maxConcurrency) {
synchronized (schedulers) {
if (schedulers.containsKey(name) == false) {
if (!schedulers.containsKey(name)) {
Scheduler s = new ParallelScheduler(name, executor, this, maxConcurrency);
schedulers.put(name, s);
executor.execute(s);
@ -86,17 +87,17 @@ public class SchedulerFactory implements SchedulerListener {
public Scheduler createOrGetRemoteScheduler(
String name,
String noteId,
RemoteInterpreterProcess interpreterProcess,
String sessionId,
RemoteInterpreter remoteInterpreter,
int maxConcurrency) {
synchronized (schedulers) {
if (schedulers.containsKey(name) == false) {
if (!schedulers.containsKey(name)) {
Scheduler s = new RemoteScheduler(
name,
executor,
noteId,
interpreterProcess,
sessionId,
remoteInterpreter,
this,
maxConcurrency);
schedulers.put(name, s);
@ -106,38 +107,24 @@ public class SchedulerFactory implements SchedulerListener {
}
}
public Scheduler removeScheduler(String name) {
public void removeScheduler(String name) {
synchronized (schedulers) {
Scheduler s = schedulers.remove(name);
if (s != null) {
s.stop();
}
}
return null;
}
public Collection<Scheduler> listScheduler(String name) {
List<Scheduler> s = new LinkedList<>();
synchronized (schedulers) {
for (Scheduler ss : schedulers.values()) {
s.add(ss);
}
}
return s;
}
@Override
public void jobStarted(Scheduler scheduler, Job job) {
logger.info("Job " + job.getJobName() + " started by scheduler " + scheduler.getName());
logger.info("Job " + job.getId() + " started by scheduler " + scheduler.getName());
}
@Override
public void jobFinished(Scheduler scheduler, Job job) {
logger.info("Job " + job.getJobName() + " finished by scheduler " + scheduler.getName());
logger.info("Job " + job.getId() + " finished by scheduler " + scheduler.getName());
}
}

1
zeppelin-interpreter/src/main/java/org/apache/zeppelin/tabledata/TableDataProxy.java
View file

@ -17,7 +17,6 @@
package org.apache.zeppelin.tabledata;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import java.util.Iterator;

76
zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/IdHashes.java Normal file
View file

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.util;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
/**
* Generate Tiny ID.
*/
public class IdHashes {
private static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9',
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
'W', 'X', 'Y', 'Z'};
/**
* encodes the given string into the base of the dictionary provided in the constructor.
*
* @param value the number to encode.
* @return the encoded string.
*/
private static String encode(Long value) {
List<Character> result = new ArrayList<>();
BigInteger base = new BigInteger("" + DICTIONARY.length);
int exponent = 1;
BigInteger remaining = new BigInteger(value.toString());
while (true) {
BigInteger a = base.pow(exponent); // 16^1 = 16
BigInteger b = remaining.mod(a); // 119 % 16 = 7 | 112 % 256 = 112
BigInteger c = base.pow(exponent - 1);
BigInteger d = b.divide(c);
// if d > dictionary.length, we have a problem. but BigInteger doesnt have
// a greater than method :-( hope for the best. theoretically, d is always
// an index of the dictionary!
result.add(DICTIONARY[d.intValue()]);
remaining = remaining.subtract(b); // 119 - 7 = 112 | 112 - 112 = 0
// finished?
if (remaining.equals(BigInteger.ZERO)) {
break;
}
exponent++;
}
// need to reverse it, since the start of the list contains the least significant values
StringBuffer sb = new StringBuffer();
for (int i = result.size() - 1; i >= 0; i--) {
sb.append(result.get(i));
}
return sb.toString();
}
public static String generateId() {
return encode(System.currentTimeMillis() + new Random().nextInt());
}
}

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/util/Util.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/util/Util.java
View file

@ -17,7 +17,7 @@
package org.apache.zeppelin.util;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
import java.util.Properties;

74
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java Normal file
View file

@ -0,0 +1,74 @@
package org.apache.zeppelin.interpreter;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import static org.mockito.Mockito.mock;
/**
* This class will load configuration files under
* src/test/resources/interpreter
* src/test/resources/conf
*
* to construct InterpreterSettingManager and InterpreterFactory properly
*
*/
public abstract class AbstractInterpreterTest {
protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractInterpreterTest.class);
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
protected InterpreterSettingManager interpreterSettingManager;
protected InterpreterFactory interpreterFactory;
protected File testRootDir;
protected File interpreterDir;
protected File confDir;
protected File notebookDir;
protected ZeppelinConfiguration conf;
@Before
public void setUp() throws Exception {
// copy the resources files to a temp folder
testRootDir = new File(System.getProperty("java.io.tmpdir") + "/Zeppelin_Test_" + System.currentTimeMillis());
testRootDir.mkdirs();
LOGGER.info("Create tmp directory: {} as root folder of ZEPPELIN_INTERPRETER_DIR & ZEPPELIN_CONF_DIR", testRootDir.getAbsolutePath());
interpreterDir = new File(testRootDir, "interpreter");
confDir = new File(testRootDir, "conf");
notebookDir = new File(testRootDir, "notebook");
interpreterDir.mkdirs();
confDir.mkdirs();
notebookDir.mkdirs();
FileUtils.copyDirectory(new File("src/test/resources/interpreter"), interpreterDir);
FileUtils.copyDirectory(new File("src/test/resources/conf"), confDir);
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_DIR.getVarName(), interpreterDir.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(), INTERPRETER_SCRIPT);
conf = new ZeppelinConfiguration();
interpreterSettingManager = new InterpreterSettingManager(conf,
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
interpreterFactory = new InterpreterFactory(interpreterSettingManager);
}
@After
public void tearDown() throws Exception {
interpreterSettingManager.close();
FileUtils.deleteDirectory(testRootDir);
}
}

40
zeppelin-server/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java
View file

@ -14,62 +14,48 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
package org.apache.zeppelin.interpreter;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
/**
*
*/
public class DoubleEchoInterpreter extends Interpreter {
public class MockInterpreter1 extends Interpreter{
Map<String, Object> vars = new HashMap<>();
public MockInterpreter1(Properties property) {
public DoubleEchoInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: "+st);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, st + "," + st);
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
return null;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return null;
}
}

43
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DummyInterpreter.java
View file

@ -1,43 +0,0 @@
package org.apache.zeppelin.interpreter;
import java.util.Properties;
/**
*
*/
public class DummyInterpreter extends Interpreter {
public DummyInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return null;
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return null;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}

57
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java
View file

@ -15,32 +15,51 @@
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.zeppelin.common.JsonSerializable;
import org.sonatype.aether.repository.RemoteRepository;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
*
* Just return the received statement back
*/
public class InterpreterInfoSaving implements JsonSerializable {
public class EchoInterpreter extends Interpreter {
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
public Map<String, InterpreterSetting> interpreterSettings;
public Map<String, List<String>> interpreterBindings;
public List<RemoteRepository> interpreterRepositories;
public String toJson() {
return gson.toJson(this);
public EchoInterpreter(Properties property) {
super(property);
}
public static InterpreterInfoSaving fromJson(String json) {
return gson.fromJson(json, InterpreterInfoSaving.class);
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
if (Boolean.parseBoolean(property.getProperty("zeppelin.interpreter.echo.fail", "false"))) {
return new InterpreterResult(InterpreterResult.Code.ERROR);
} else {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, st);
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}

66
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java Normal file
View file

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.junit.Test;
import java.io.IOException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class InterpreterFactoryTest extends AbstractInterpreterTest {
@Test
public void testGetFactory() throws IOException {
// no default interpreter because there's no interpreter setting binded to this note
assertNull(interpreterFactory.getInterpreter("user1", "note1", ""));
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "") instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "");
// EchoInterpreter is the default interpreter (see zeppelin-interpreter/src/test/resources/conf/interpreter.json)
assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "test") instanceof RemoteInterpreter);
remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "test");
assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "echo") instanceof RemoteInterpreter);
remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "echo");
assertEquals(EchoInterpreter.class.getName(), remoteInterpreter.getClassName());
assertTrue(interpreterFactory.getInterpreter("user1", "note1", "double_echo") instanceof RemoteInterpreter);
remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("user1", "note1", "double_echo");
assertEquals(DoubleEchoInterpreter.class.getName(), remoteInterpreter.getClassName());
}
@Test(expected = InterpreterException.class)
public void testUnknownRepl1() throws IOException {
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl");
}
@Test
public void testUnknownRepl2() throws IOException {
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
assertNull(interpreterFactory.getInterpreter("user1", "note1", "unknown_repl"));
}
}

90
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterGroupTest.java Normal file
View file

@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static org.junit.Assert.assertEquals;
public class InterpreterGroupTest {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroupTest.class);
private InterpreterSetting interpreterSetting;
@Before
public void setUp() throws IOException, RepositoryException {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
interpreterOption.setRemote(false);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
}
@Test
public void testInterpreterGroup() {
InterpreterGroup interpreterGroup = new InterpreterGroup("group_1", interpreterSetting);
assertEquals(0, interpreterGroup.getSessionNum());
// create session_1
List<Interpreter> interpreters = interpreterGroup.getOrCreateSession("user1", "session_1");
assertEquals(2, interpreters.size());
assertEquals(EchoInterpreter.class.getName(), interpreters.get(0).getClassName());
assertEquals(DoubleEchoInterpreter.class.getName(), interpreters.get(1).getClassName());
assertEquals(1, interpreterGroup.getSessionNum());
// get the same interpreters when interpreterGroup.getOrCreateSession is invoked again
assertEquals(interpreters, interpreterGroup.getOrCreateSession("user1", "session_1"));
assertEquals(1, interpreterGroup.getSessionNum());
// create session_2
List<Interpreter> interpreters2 = interpreterGroup.getOrCreateSession("user1", "session_2");
assertEquals(2, interpreters2.size());
assertEquals(EchoInterpreter.class.getName(), interpreters2.get(0).getClassName());
assertEquals(DoubleEchoInterpreter.class.getName(), interpreters2.get(1).getClassName());
assertEquals(2, interpreterGroup.getSessionNum());
// close session_1
interpreterGroup.close("session_1");
assertEquals(1, interpreterGroup.getSessionNum());
// close InterpreterGroup
interpreterGroup.close();
assertEquals(0, interpreterGroup.getSessionNum());
}
}

11
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterOutputChangeWatcherTest.java
View file

@ -21,6 +21,7 @@ import static org.junit.Assert.*;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.After;
import org.junit.Before;
@ -29,7 +30,7 @@ import org.junit.Test;
public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChangeListener {
private File tmpDir;
private File fileChanged;
private int numChanged;
private AtomicInteger numChanged;
private InterpreterOutputChangeWatcher watcher;
@Before
@ -40,7 +41,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
fileChanged = null;
numChanged = 0;
numChanged = new AtomicInteger(0);
}
@After
@ -66,7 +67,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
@Test
public void test() throws IOException, InterruptedException {
assertNull(fileChanged);
assertEquals(0, numChanged);
assertEquals(0, numChanged.get());
Thread.sleep(1000);
// create new file
@ -92,14 +93,14 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan
}
assertNotNull(fileChanged);
assertEquals(1, numChanged);
assertEquals(1, numChanged.get());
}
@Override
public void fileChanged(File file) {
fileChanged = file;
numChanged++;
numChanged.incrementAndGet();
synchronized(this) {
notify();

270
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java Normal file
View file

@ -0,0 +1,270 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.junit.Test;
import org.sonatype.aether.RepositoryException;
import org.sonatype.aether.repository.RemoteRepository;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
public class InterpreterSettingManagerTest extends AbstractInterpreterTest {
@Test
public void testInitInterpreterSettingManager() throws IOException, RepositoryException {
assertEquals(2, interpreterSettingManager.get().size());
InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test");
assertEquals("test", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
assertEquals(2, interpreterSetting.getInterpreterInfos().size());
// 3 other builtin properties:
// * zeppelin.interpeter.output.limit
// * zeppelin.interpreter.localRepo
// * zeppelin.interpreter.max.poolsize
assertEquals(6, interpreterSetting.getJavaProperties().size());
assertEquals("value_1", interpreterSetting.getJavaProperties().getProperty("property_1"));
assertEquals("new_value_2", interpreterSetting.getJavaProperties().getProperty("property_2"));
assertEquals("value_3", interpreterSetting.getJavaProperties().getProperty("property_3"));
assertEquals("shared", interpreterSetting.getOption().perNote);
assertEquals("shared", interpreterSetting.getOption().perUser);
assertEquals(0, interpreterSetting.getDependencies().size());
assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
assertNotNull(interpreterSetting.getAppEventListener());
assertNotNull(interpreterSetting.getDependencyResolver());
assertNotNull(interpreterSetting.getInterpreterSettingManager());
List<RemoteRepository> repositories = interpreterSettingManager.getRepositories();
assertEquals(2, repositories.size());
assertEquals("central", repositories.get(0).getId());
// Load it again
InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf,
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
assertEquals(2, interpreterSettingManager2.get().size());
interpreterSetting = interpreterSettingManager2.getByName("test");
assertEquals("test", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
assertEquals(2, interpreterSetting.getInterpreterInfos().size());
assertEquals(6, interpreterSetting.getJavaProperties().size());
assertEquals("value_1", interpreterSetting.getJavaProperties().getProperty("property_1"));
assertEquals("new_value_2", interpreterSetting.getJavaProperties().getProperty("property_2"));
assertEquals("value_3", interpreterSetting.getJavaProperties().getProperty("property_3"));
assertEquals("shared", interpreterSetting.getOption().perNote);
assertEquals("shared", interpreterSetting.getOption().perUser);
assertEquals(0, interpreterSetting.getDependencies().size());
repositories = interpreterSettingManager2.getRepositories();
assertEquals(2, repositories.size());
assertEquals("central", repositories.get(0).getId());
}
@Test
public void testCreateUpdateRemoveSetting() throws IOException {
// create new interpreter setting
InterpreterOption option = new InterpreterOption();
option.setPerNote("scoped");
option.setPerUser("scoped");
Map<String, InterpreterProperty> properties = new HashMap<>();
properties.put("property_4", new InterpreterProperty("property_4","value_4"));
try {
interpreterSettingManager.createNewSetting("test2", "test", new ArrayList<Dependency>(), option, properties);
fail("Should fail due to interpreter already existed");
} catch (IOException e) {
assertTrue(e.getMessage().contains("already existed"));
}
interpreterSettingManager.createNewSetting("test3", "test", new ArrayList<Dependency>(), option, properties);
assertEquals(3, interpreterSettingManager.get().size());
InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("test3");
assertEquals("test3", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
// 3 other builtin properties:
// * zeppelin.interpeter.output.limit
// * zeppelin.interpreter.localRepo
// * zeppelin.interpreter.max.poolsize
assertEquals(4, interpreterSetting.getJavaProperties().size());
assertEquals("value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
assertEquals("scoped", interpreterSetting.getOption().perNote);
assertEquals("scoped", interpreterSetting.getOption().perUser);
assertEquals(0, interpreterSetting.getDependencies().size());
assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
assertNotNull(interpreterSetting.getAppEventListener());
assertNotNull(interpreterSetting.getDependencyResolver());
assertNotNull(interpreterSetting.getInterpreterSettingManager());
// load it again, it should be saved in interpreter-setting.json. So we can restore it properly
InterpreterSettingManager interpreterSettingManager2 = new InterpreterSettingManager(conf,
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
assertEquals(3, interpreterSettingManager2.get().size());
interpreterSetting = interpreterSettingManager2.getByName("test3");
assertEquals("test3", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
assertEquals(6, interpreterSetting.getJavaProperties().size());
assertEquals("value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
assertEquals("scoped", interpreterSetting.getOption().perNote);
assertEquals("scoped", interpreterSetting.getOption().perUser);
assertEquals(0, interpreterSetting.getDependencies().size());
// update interpreter setting
InterpreterOption newOption = new InterpreterOption();
newOption.setPerNote("scoped");
newOption.setPerUser("isolated");
Map<String, InterpreterProperty> newProperties = new HashMap<>(properties);
newProperties.put("property_4", new InterpreterProperty("property_4", "new_value_4"));
List<Dependency> newDependencies = new ArrayList<>();
newDependencies.add(new Dependency("com.databricks:spark-avro_2.11:3.1.0"));
interpreterSettingManager.setPropertyAndRestart(interpreterSetting.getId(), newOption, newProperties, newDependencies);
interpreterSetting = interpreterSettingManager.get(interpreterSetting.getId());
assertEquals("test3", interpreterSetting.getName());
assertEquals("test", interpreterSetting.getGroup());
assertEquals(4, interpreterSetting.getJavaProperties().size());
assertEquals("new_value_4", interpreterSetting.getJavaProperties().getProperty("property_4"));
assertEquals("scoped", interpreterSetting.getOption().perNote);
assertEquals("isolated", interpreterSetting.getOption().perUser);
assertEquals(1, interpreterSetting.getDependencies().size());
assertNotNull(interpreterSetting.getAngularObjectRegistryListener());
assertNotNull(interpreterSetting.getRemoteInterpreterProcessListener());
assertNotNull(interpreterSetting.getAppEventListener());
assertNotNull(interpreterSetting.getDependencyResolver());
assertNotNull(interpreterSetting.getInterpreterSettingManager());
// restart in note page
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds());
interpreterSettingManager.setInterpreterBinding("user2", "note2", interpreterSettingManager.getSettingIds());
interpreterSettingManager.setInterpreterBinding("user3", "note3", interpreterSettingManager.getSettingIds());
// create 3 sessions as it is scoped mode
interpreterSetting.getOption().setPerUser("scoped");
interpreterSetting.getDefaultInterpreter("user1", "note1");
interpreterSetting.getDefaultInterpreter("user2", "note2");
interpreterSetting.getDefaultInterpreter("user3", "note3");
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
assertEquals(3, interpreterGroup.getSessionNum());
// only close user1's session
interpreterSettingManager.restart(interpreterSetting.getId(), "note1", "user1");
assertEquals(2, interpreterGroup.getSessionNum());
// close all the sessions
interpreterSettingManager.restart(interpreterSetting.getId(), "note1", "anonymous");
assertEquals(0, interpreterGroup.getSessionNum());
// remove interpreter setting
interpreterSettingManager.remove(interpreterSetting.getId());
assertEquals(2, interpreterSettingManager.get().size());
// load it again
InterpreterSettingManager interpreterSettingManager3 = new InterpreterSettingManager(new ZeppelinConfiguration(),
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
assertEquals(2, interpreterSettingManager3.get().size());
}
@Test
public void testInterpreterBinding() throws IOException {
assertNull(interpreterSettingManager.getInterpreterBinding("note1"));
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
assertEquals(interpreterSettingManager.getInterpreterSettingIds(), interpreterSettingManager.getInterpreterBinding("note1"));
}
@Test
public void testUpdateInterpreterBinding_PerNoteShared() throws IOException {
InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
defaultInterpreterSetting.getOption().setPerNote("shared");
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
// create interpreter of the first binded interpreter setting
interpreterFactory.getInterpreter("user1", "note1", "");
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
// choose the first setting
List<String> newSettingIds = new ArrayList<>();
newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
// InterpreterGroup will still be alive as it is shared
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testUpdateInterpreterBinding_PerNoteIsolated() throws IOException {
InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
defaultInterpreterSetting.getOption().setPerNote("isolated");
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
// create interpreter of the first binded interpreter setting
interpreterFactory.getInterpreter("user1", "note1", "");
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
// choose the first setting
List<String> newSettingIds = new ArrayList<>();
newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
// InterpreterGroup will be closed as it is only belong to this note
assertEquals(0, defaultInterpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testUpdateInterpreterBinding_PerNoteScoped() throws IOException {
InterpreterSetting defaultInterpreterSetting = interpreterSettingManager.get().get(0);
defaultInterpreterSetting.getOption().setPerNote("scoped");
interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getInterpreterSettingIds());
interpreterSettingManager.setInterpreterBinding("user1", "note2", interpreterSettingManager.getInterpreterSettingIds());
// create 2 interpreter of the first binded interpreter setting for note1 and note2
interpreterFactory.getInterpreter("user1", "note1", "");
interpreterFactory.getInterpreter("user1", "note2", "");
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, defaultInterpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// choose the first setting
List<String> newSettingIds = new ArrayList<>();
newSettingIds.add(interpreterSettingManager.getInterpreterSettingIds().get(1));
interpreterSettingManager.setInterpreterBinding("user1", "note1", newSettingIds);
assertEquals(newSettingIds, interpreterSettingManager.getInterpreterBinding("note1"));
// InterpreterGroup will be still alive but session belong to note1 will be closed
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, defaultInterpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
}
}

411
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java Normal file
View file

@ -0,0 +1,411 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.junit.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
public class InterpreterSettingTest {
@Test
public void testCreateInterpreters() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SHARED);
interpreterOption.setRemote(false);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create default interpreter for user1 and note1
assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getDefaultInterpreter("user1", "note1").getClassName());
// create interpreter echo for user1 and note1
assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "echo").getClassName());
assertEquals(interpreterSetting.getDefaultInterpreter("user1", "note1"), interpreterSetting.getInterpreter("user1", "note1", "echo"));
// create interpreter double_echo for user1 and note1
assertEquals(DoubleEchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "double_echo").getClassName());
// create non-existed interpreter
assertNull(interpreterSetting.getInterpreter("user1", "note1", "invalid_echo"));
}
@Test
public void testSharedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SHARED);
interpreterOption.setRemote(false);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create default interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// create default interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// create default interpreter user1 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// only 1 session is created, this session is shared across users and notes
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeInterpreters("note1", "user1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testPerUserScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeInterpreters("user1", "note1");
// InterpreterGroup is still there, but one session is removed
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.SCOPED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user1 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeInterpreters("user1", "note1");
// InterpreterGroup is still there, but one session is removed
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testPerUserIsolatedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
// Each user own one InterpreterGroup and one session per InterpreterGroup
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(1).getSessionNum());
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testPerNoteIsolatedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.ISOLATED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
// Each user own one InterpreterGroup and one session per InterpreterGroup
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(1).getSessionNum());
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testPerUserIsolatedPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
interpreterOption.setPerNote(InterpreterOption.SCOPED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
// group1 for user1 has 2 sessions, and group2 for user2 has 1 session
assertEquals(interpreterSetting.getInterpreterGroup("user1", "note1"), interpreterSetting.getInterpreterGroup("user1", "note2"));
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").getSessionNum());
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note2").getSessionNum());
assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").getSessionNum());
// close one session for user1
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").getSessionNum());
// close another session for user1
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// close session for user2
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testPerUserIsolatedPerNoteIsolatedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
interpreterOption.setPerNote(InterpreterOption.ISOLATED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// create interpreter for user1 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(3, interpreterSetting.getAllInterpreterGroups().size());
// create interpreter for user2 and note2
interpreterSetting.getDefaultInterpreter("user2", "note2");
assertEquals(4, interpreterSetting.getAllInterpreterGroups().size());
for (InterpreterGroup interpreterGroup : interpreterSetting.getAllInterpreterGroups()) {
// each InterpreterGroup has one session
assertEquals(1, interpreterGroup.getSessionNum());
}
// close one session for user1 and note1
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(3, interpreterSetting.getAllInterpreterGroups().size());
// close one session for user1 and note2
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
// close one session for user2 and note1
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// close one session for user2 and note2
interpreterSetting.closeInterpreters("user2", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void testPerUserScopedPerNoteScopedMode() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
interpreterOption.setPerNote(InterpreterOption.SCOPED);
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
InterpreterSetting interpreterSetting = new InterpreterSetting.Builder()
.setId("id")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.create();
// create interpreter for user1 and note1
interpreterSetting.getDefaultInterpreter("user1", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user1 and note2
interpreterSetting.getDefaultInterpreter("user1", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note1
interpreterSetting.getDefaultInterpreter("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(3, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// create interpreter for user2 and note2
interpreterSetting.getDefaultInterpreter("user2", "note2");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(4, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// close one session for user1 and note1
interpreterSetting.closeInterpreters("user1", "note1");
assertEquals(3, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// close one session for user1 and note2
interpreterSetting.closeInterpreters("user1", "note2");
assertEquals(2, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// close one session for user2 and note1
interpreterSetting.closeInterpreters("user2", "note1");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
// close one session for user2 and note2
interpreterSetting.closeInterpreters("user2", "note2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
}

7
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java
View file

@ -24,13 +24,14 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
//TODO(zjffdu) add more test for Interpreter which is a very important class
public class InterpreterTest {
@Test
public void testDefaultProperty() {
Properties p = new Properties();
p.put("p1", "v1");
Interpreter intp = new DummyInterpreter(p);
Interpreter intp = new EchoInterpreter(p);
assertEquals(1, intp.getProperty().size());
assertEquals("v1", intp.getProperty().get("p1"));
@ -41,7 +42,7 @@ public class InterpreterTest {
public void testOverriddenProperty() {
Properties p = new Properties();
p.put("p1", "v1");
Interpreter intp = new DummyInterpreter(p);
Interpreter intp = new EchoInterpreter(p);
Properties overriddenProperty = new Properties();
overriddenProperty.put("p1", "v2");
intp.setProperty(overriddenProperty);
@ -73,7 +74,7 @@ public class InterpreterTest {
Properties p = new Properties();
p.put("p1", "replName #{noteId}, #{paragraphTitle}, #{paragraphId}, #{paragraphText}, #{replName}, #{noteId}, #{user}," +
" #{authenticationInfo}");
Interpreter intp = new DummyInterpreter(p);
Interpreter intp = new EchoInterpreter(p);
intp.setUserName(user);
String actual = intp.getProperty("p1");
InterpreterContext.remove();

60
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java Normal file
View file

@ -0,0 +1,60 @@
package org.apache.zeppelin.interpreter;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import java.util.Properties;
/**
* Interpreter that only accept long value and sleep for such period
*/
public class SleepInterpreter extends Interpreter {
public SleepInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
try {
Thread.sleep(Long.parseLong(st));
return new InterpreterResult(InterpreterResult.Code.SUCCESS);
} catch (Exception e) {
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.NATIVE;
}
@Override
public Scheduler getScheduler() {
if (Boolean.parseBoolean(property.getProperty("zeppelin.SleepInterpreter.parallel", "false"))) {
return SchedulerFactory.singleton().createOrGetParallelScheduler(
"Parallel-" + SleepInterpreter.class.getName(), 10);
}
return super.getScheduler();
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
}

0
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java
View file

15
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java
View file

@ -17,11 +17,6 @@
package org.apache.zeppelin.interpreter.mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -29,8 +24,14 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
public class MockInterpreter1 extends Interpreter{
Map<String, Object> vars = new HashMap<>();
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
public class MockInterpreter1 extends Interpreter {
Map<String, Object> vars = new HashMap<>();
public MockInterpreter1(Properties property) {
super(property);

20
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter11.java
View file

@ -17,10 +17,6 @@
package org.apache.zeppelin.interpreter.mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
@ -29,12 +25,18 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
public class MockInterpreter11 extends Interpreter{
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
public class MockInterpreter11 extends Interpreter {
Map<String, Object> vars = new HashMap<>();
public MockInterpreter11(Properties property) {
super(property);
}
boolean open;
@Override
@ -53,7 +55,7 @@ public class MockInterpreter11 extends Interpreter{
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl11: "+st);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl11: " + st);
}
@Override
@ -72,12 +74,12 @@ public class MockInterpreter11 extends Interpreter{
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_"+this.hashCode());
return SchedulerFactory.singleton().createOrGetFIFOScheduler("test_" + this.hashCode());
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
InterpreterContext interpreterContext) {
return null;
}
}
}

10
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter2.java
View file

@ -17,11 +17,6 @@
package org.apache.zeppelin.interpreter.mock;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
@ -29,6 +24,11 @@ import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
public class MockInterpreter2 extends Interpreter{
Map<String, Object> vars = new HashMap<>();

30
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/AppendOutputRunnerTest.java
View file

@ -17,23 +17,6 @@
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.atMost;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@ -43,6 +26,19 @@ import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.*;
public class AppendOutputRunnerTest {
private static final int NUM_EVENTS = 10000;

87
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
View file

@ -17,15 +17,10 @@
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.zeppelin.display.*;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular;
import org.apache.zeppelin.resource.LocalResourcePool;
@ -34,17 +29,25 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup;
private HashMap<String, String> env;
private RemoteInterpreter intp;
private InterpreterContext context;
private RemoteAngularObjectRegistry localRegistry;
private InterpreterSetting interpreterSetting;
private AtomicInteger onAdd;
private AtomicInteger onUpdate;
@ -56,32 +59,24 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
onUpdate = new AtomicInteger(0);
onRemove = new AtomicInteger(0);
intpGroup = new InterpreterGroup("intpId");
localRegistry = new RemoteAngularObjectRegistry("intpId", this, intpGroup);
intpGroup.setAngularObjectRegistry(localRegistry);
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterAngular.class.getName(), "mock", true, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
interpreterSetting = new InterpreterSetting.Builder()
.setId("test")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.setRunner(runner)
.setInterpreterDir("../interpeters/test")
.create();
Properties p = new Properties();
intp = new RemoteInterpreter(
p,
"note",
MockInterpreterAngular.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false
);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intp);
intp.setInterpreterGroup(intpGroup);
intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
localRegistry = (RemoteAngularObjectRegistry) intp.getInterpreterGroup().getAngularObjectRegistry();
context = new InterpreterContext(
"note",
@ -92,17 +87,17 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new AngularObjectRegistry(intp.getInterpreterGroup().getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
intp.open();
}
@After
public void tearDown() throws Exception {
intp.close();
intpGroup.close();
interpreterSetting.close();
}
@Test
@ -147,7 +142,7 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
Thread.sleep(500); // waitFor eventpoller pool event
String[] result = ret.message().get(0).getData().split(" ");
assertEquals("0", result[0]); // size of registry
// create object
ret = intp.interpret("add n1 v1", context);
Thread.sleep(500);
@ -172,11 +167,11 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
Thread.sleep(500); // waitFor eventpoller pool event
String[] result = ret.message().get(0).getData().split(" ");
assertEquals("0", result[0]); // size of registry
// create object
localRegistry.addAndNotifyRemoteProcess("n1", "v1", "note", null);
// get from remote registry
// get from remote registry
ret = intp.interpret("get", context);
Thread.sleep(500); // waitFor eventpoller pool event
result = ret.message().get(0).getData().split(" ");

0
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPollerTest.java
View file

65
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java
View file

@ -17,22 +17,18 @@
package org.apache.zeppelin.interpreter.remote;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterOutputStream;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Properties;
import java.util.*;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
/**
@ -43,41 +39,32 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup;
private HashMap<String, String> env;
private InterpreterSetting interpreterSetting;
@Before
public void setUp() throws Exception {
intpGroup = new InterpreterGroup();
intpGroup.put("note", new LinkedList<Interpreter>());
InterpreterOption interpreterOption = new InterpreterOption();
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterOutputStream.class.getName(), "mock", true, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
interpreterSetting = new InterpreterSetting.Builder()
.setId("test")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.setRunner(runner)
.setInterpreterDir("../interpeters/test")
.create();
}
@After
public void tearDown() throws Exception {
intpGroup.close();
}
private RemoteInterpreter createMockInterpreter() {
RemoteInterpreter intp = new RemoteInterpreter(
new Properties(),
"note",
MockInterpreterOutputStream.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
this,
null,
"anonymous",
false);
intpGroup.get("note").add(intp);
intp.setInterpreterGroup(intpGroup);
return intp;
interpreterSetting.close();
}
private InterpreterContext createInterpreterContext() {
@ -90,14 +77,14 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
null,
new LinkedList<InterpreterContextRunner>(), null);
}
@Test
public void testInterpreterResultOnly() {
RemoteInterpreter intp = createMockInterpreter();
RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS::staticresult", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("staticresult", ret.message().get(0).getData());
@ -113,7 +100,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testInterpreterOutputStreamOnly() {
RemoteInterpreter intp = createMockInterpreter();
RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:streamresult:", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("streamresult", ret.message().get(0).getData());
@ -125,7 +112,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testInterpreterResultOutputStreamMixed() {
RemoteInterpreter intp = createMockInterpreter();
RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:stream:static", createInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
assertEquals("stream", ret.message().get(0).getData());
@ -134,7 +121,7 @@ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProce
@Test
public void testOutputType() {
RemoteInterpreter intp = createMockInterpreter();
RemoteInterpreter intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
InterpreterResult ret = intp.interpret("SUCCESS:%html hello:", createInterpreterContext());
assertEquals(InterpreterResult.Type.HTML, ret.message().get(0).getType());

520
zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java Normal file
View file

@ -0,0 +1,520 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import org.apache.thrift.transport.TTransportException;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.remote.mock.GetEnvPropertyInterpreter;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
public class RemoteInterpreterTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterSetting interpreterSetting;
@Before
public void setUp() throws Exception {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap<String, Object>());
InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap<String, Object>());
InterpreterInfo interpreterInfo3 = new InterpreterInfo(SleepInterpreter.class.getName(), "sleep", false, new HashMap<String, Object>());
InterpreterInfo interpreterInfo4 = new InterpreterInfo(GetEnvPropertyInterpreter.class.getName(), "get", false, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
interpreterInfos.add(interpreterInfo2);
interpreterInfos.add(interpreterInfo3);
interpreterInfos.add(interpreterInfo4);
InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
interpreterSetting = new InterpreterSetting.Builder()
.setId("test")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.setRunner(runner)
.setInterpreterDir("../interpeters/test")
.create();
}
@After
public void tearDown() throws Exception {
interpreterSetting.close();
}
@Test
public void testSharedMode() {
interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
assertTrue(interpreter1 instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
assertTrue(interpreter2 instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
"title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
null, null, new ArrayList<InterpreterContextRunner>(), null);
assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
assertEquals(0, remoteInterpreter1.getProgress(context1));
assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
assertEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
// Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
// RemoteInterpreterProcess leakage.
remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
assertNull(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess());
try {
assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
fail("Should not be able to call interpret after interpreter is closed");
} catch (Exception e) {
e.printStackTrace();
}
try {
assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
fail("Should not be able to call getProgress after RemoterInterpreterProcess is stoped");
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void testScopedMode() {
interpreterSetting.getOption().setPerUser(InterpreterOption.SCOPED);
Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
assertTrue(interpreter1 instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
assertTrue(interpreter2 instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
"title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
null, null, new ArrayList<InterpreterContextRunner>(), null);
assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
assertEquals(0, remoteInterpreter1.getProgress(context1));
assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
assertEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
// Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
// RemoteInterpreterProcess leakage.
remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
try {
assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
fail("Should not be able to call interpret after interpreter is closed");
} catch (Exception e) {
e.printStackTrace();
}
assertTrue(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
remoteInterpreter2.getInterpreterGroup().close(remoteInterpreter2.getSessionId());
try {
assertEquals("hello", remoteInterpreter2.interpret("hello", context1));
fail("Should not be able to call interpret after interpreter is closed");
} catch (Exception e) {
e.printStackTrace();
}
assertNull(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
}
@Test
public void testIsolatedMode() {
interpreterSetting.getOption().setPerUser(InterpreterOption.ISOLATED);
Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
Interpreter interpreter2 = interpreterSetting.getDefaultInterpreter("user2", "note1");
assertTrue(interpreter1 instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
assertTrue(interpreter2 instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter2 = (RemoteInterpreter) interpreter2;
InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
"title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
null, null, new ArrayList<InterpreterContextRunner>(), null);
assertEquals("hello", remoteInterpreter1.interpret("hello", context1).message().get(0).getData());
assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
assertEquals(Interpreter.FormType.NATIVE, interpreter1.getFormType());
assertEquals(0, remoteInterpreter1.getProgress(context1));
assertNotNull(remoteInterpreter1.getOrCreateInterpreterProcess());
assertTrue(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
assertNotEquals(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess(),
remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
// Call InterpreterGroup.close instead of Interpreter.close, otherwise we will have the
// RemoteInterpreterProcess leakage.
remoteInterpreter1.getInterpreterGroup().close(remoteInterpreter1.getSessionId());
assertNull(remoteInterpreter1.getInterpreterGroup().getRemoteInterpreterProcess());
assertTrue(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess().isRunning());
try {
remoteInterpreter1.interpret("hello", context1);
fail("Should not be able to call getProgress after interpreter is closed");
} catch (Exception e) {
e.printStackTrace();
}
assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
remoteInterpreter2.getInterpreterGroup().close(remoteInterpreter2.getSessionId());
try {
assertEquals("hello", remoteInterpreter2.interpret("hello", context1).message().get(0).getData());
fail("Should not be able to call interpret after interpreter is closed");
} catch (Exception e) {
e.printStackTrace();
}
assertNull(remoteInterpreter2.getInterpreterGroup().getRemoteInterpreterProcess());
}
// @Test
// public void testExecuteIncorrectPrecode() throws TTransportException, IOException {
// interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
// interpreterSetting.getProperties().setProperty("zeppelin.SleepInterpreter.precode", "fail test");
//
// Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
// InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
// "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
// null, null, new ArrayList<InterpreterContextRunner>(), null);
// assertEquals(Code.ERROR, interpreter1.interpret("10", context1).code());
// }
//
// @Test
// public void testExecuteCorrectPrecode() throws TTransportException, IOException {
// interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
// interpreterSetting.getProperties().setProperty("zeppelin.SleepInterpreter.precode", "1");
//
// Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
// InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
// "title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
// null, null, new ArrayList<InterpreterContextRunner>(), null);
// assertEquals(Code.SUCCESS, interpreter1.interpret("10", context1).code());
// }
@Test
public void testRemoteInterperterErrorStatus() throws TTransportException, IOException {
interpreterSetting.setProperty("zeppelin.interpreter.echo.fail", "true");
interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
Interpreter interpreter1 = interpreterSetting.getDefaultInterpreter("user1", "note1");
assertTrue(interpreter1 instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter1 = (RemoteInterpreter) interpreter1;
InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
"title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
null, null, new ArrayList<InterpreterContextRunner>(), null);
assertEquals(Code.ERROR, remoteInterpreter1.interpret("hello", context1).code());
}
@Test
public void testFIFOScheduler() throws InterruptedException {
interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
// by default SleepInterpreter would use FIFOScheduler
final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
"title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
null, null, new ArrayList<InterpreterContextRunner>(), null);
// run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
// time overhead of launching the process.
interpreter1.interpret("1", context1);
Thread thread1 = new Thread() {
@Override
public void run() {
assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
}
};
Thread thread2 = new Thread() {
@Override
public void run() {
assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
}
};
long start = System.currentTimeMillis();
thread1.start();
thread2.start();
thread1.join();
thread2.join();
long end = System.currentTimeMillis();
assertTrue((end - start) >= 200);
}
@Test
public void testParallelScheduler() throws InterruptedException {
interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
interpreterSetting.setProperty("zeppelin.SleepInterpreter.parallel", "true");
final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "sleep");
final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
"title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
null, null, new ArrayList<InterpreterContextRunner>(), null);
// run this dummy interpret method first to launch the RemoteInterpreterProcess to avoid the
// time overhead of launching the process.
interpreter1.interpret("1", context1);
Thread thread1 = new Thread() {
@Override
public void run() {
assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
}
};
Thread thread2 = new Thread() {
@Override
public void run() {
assertEquals(Code.SUCCESS, interpreter1.interpret("100", context1).code());
}
};
long start = System.currentTimeMillis();
thread1.start();
thread2.start();
thread1.join();
thread2.join();
long end = System.currentTimeMillis();
assertTrue((end - start) <= 200);
}
// @Test
// public void testRunOrderPreserved() throws InterruptedException {
// Properties p = new Properties();
// intpGroup.put("note", new LinkedList<Interpreter>());
//
// final RemoteInterpreter intpA = createMockInterpreterA(p);
//
// intpGroup.get("note").add(intpA);
// intpA.setInterpreterGroup(intpGroup);
//
// intpA.open();
//
// int concurrency = 3;
// final List<InterpreterResultMessage> results = new LinkedList<>();
//
// Scheduler scheduler = intpA.getScheduler();
// for (int i = 0; i < concurrency; i++) {
// final String jobId = Integer.toString(i);
// scheduler.submit(new Job(jobId, Integer.toString(i), null, 200) {
// private Object r;
//
// @Override
// public Object getReturn() {
// return r;
// }
//
// @Override
// public void setResult(Object results) {
// this.r = results;
// }
//
// @Override
// public int progress() {
// return 0;
// }
//
// @Override
// public Map<String, Object> info() {
// return null;
// }
//
// @Override
// protected Object jobRun() throws Throwable {
// InterpreterResult ret = intpA.interpret(getJobName(), new InterpreterContext(
// "note",
// jobId,
// null,
// "title",
// "text",
// new AuthenticationInfo(),
// new HashMap<String, Object>(),
// new GUI(),
// new AngularObjectRegistry(intpGroup.getId(), null),
// new LocalResourcePool("pool1"),
// new LinkedList<InterpreterContextRunner>(), null));
//
// synchronized (results) {
// results.addAll(ret.message());
// results.notify();
// }
// return null;
// }
//
// @Override
// protected boolean jobAbort() {
// return false;
// }
//
// });
// }
//
// // wait for job finished
// synchronized (results) {
// while (results.size() != concurrency) {
// results.wait(300);
// }
// }
//
// int i = 0;
// for (InterpreterResultMessage result : results) {
// assertEquals(Integer.toString(i++), result.getData());
// }
// assertEquals(concurrency, i);
//
// intpA.close();
// }
// @Test
// public void testRemoteInterpreterSharesTheSameSchedulerInstanceInTheSameGroup() {
// Properties p = new Properties();
// intpGroup.put("note", new LinkedList<Interpreter>());
//
// RemoteInterpreter intpA = createMockInterpreterA(p);
//
// intpGroup.get("note").add(intpA);
// intpA.setInterpreterGroup(intpGroup);
//
// RemoteInterpreter intpB = createMockInterpreterB(p);
//
// intpGroup.get("note").add(intpB);
// intpB.setInterpreterGroup(intpGroup);
//
// intpA.open();
// intpB.open();
//
// assertEquals(intpA.getScheduler(), intpB.getScheduler());
// }
// @Test
// public void testMultiInterpreterSession() {
// Properties p = new Properties();
// intpGroup.put("sessionA", new LinkedList<Interpreter>());
// intpGroup.put("sessionB", new LinkedList<Interpreter>());
//
// RemoteInterpreter intpAsessionA = createMockInterpreterA(p, "sessionA");
// intpGroup.get("sessionA").add(intpAsessionA);
// intpAsessionA.setInterpreterGroup(intpGroup);
//
// RemoteInterpreter intpBsessionA = createMockInterpreterB(p, "sessionA");
// intpGroup.get("sessionA").add(intpBsessionA);
// intpBsessionA.setInterpreterGroup(intpGroup);
//
// intpAsessionA.open();
// intpBsessionA.open();
//
// assertEquals(intpAsessionA.getScheduler(), intpBsessionA.getScheduler());
//
// RemoteInterpreter intpAsessionB = createMockInterpreterA(p, "sessionB");
// intpGroup.get("sessionB").add(intpAsessionB);
// intpAsessionB.setInterpreterGroup(intpGroup);
//
// RemoteInterpreter intpBsessionB = createMockInterpreterB(p, "sessionB");
// intpGroup.get("sessionB").add(intpBsessionB);
// intpBsessionB.setInterpreterGroup(intpGroup);
//
// intpAsessionB.open();
// intpBsessionB.open();
//
// assertEquals(intpAsessionB.getScheduler(), intpBsessionB.getScheduler());
// assertNotEquals(intpAsessionA.getScheduler(), intpAsessionB.getScheduler());
// }
// @Test
// public void should_push_local_angular_repo_to_remote() throws Exception {
// //Given
// final Client client = mock(Client.class);
// final RemoteInterpreter intr = null;
//// new RemoteInterpreter(new Properties(), "noteId",
//// MockInterpreterA.class.getName(), "runner", "path", "localRepo", env, 10 * 1000, null,
//// null, "anonymous", false);
// final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null);
// registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId");
// final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId");
// interpreterGroup.setAngularObjectRegistry(registry);
// intr.setInterpreterGroup(interpreterGroup);
//
// final java.lang.reflect.Type registryType = new TypeToken<Map<String,
// Map<String, AngularObject>>>() {
// }.getType();
// final Gson gson = new Gson();
// final String expected = gson.toJson(registry.getRegistry(), registryType);
//
// //When
//// intr.pushAngularObjectRegistryToRemote(client);
//
// //Then
// Mockito.verify(client).angularRegistryPush(expected);
// }
@Test
public void testEnvStringPattern() {
assertFalse(RemoteInterpreterUtils.isEnvString(null));
assertFalse(RemoteInterpreterUtils.isEnvString(""));
assertFalse(RemoteInterpreterUtils.isEnvString("abcDEF"));
assertFalse(RemoteInterpreterUtils.isEnvString("ABC-DEF"));
assertTrue(RemoteInterpreterUtils.isEnvString("ABCDEF"));
assertTrue(RemoteInterpreterUtils.isEnvString("ABC_DEF"));
assertTrue(RemoteInterpreterUtils.isEnvString("ABC_DEF123"));
}
@Test
public void testEnvironmentAndProperty() {
interpreterSetting.getOption().setPerUser(InterpreterOption.SHARED);
interpreterSetting.setProperty("ENV_1", "VALUE_1");
interpreterSetting.setProperty("property_1", "value_1");
final Interpreter interpreter1 = interpreterSetting.getInterpreter("user1", "note1", "get");
final InterpreterContext context1 = new InterpreterContext("noteId", "paragraphId", "repl",
"title", "text", AuthenticationInfo.ANONYMOUS, new HashMap<String, Object>(), new GUI(),
null, null, new ArrayList<InterpreterContextRunner>(), null);
assertEquals("VALUE_1", interpreter1.interpret("getEnv ENV_1", context1).message().get(0).getData());
assertEquals("null", interpreter1.interpret("getEnv ENV_2", context1).message().get(0).getData());
assertEquals("value_1", interpreter1.interpret("getProperty property_1", context1).message().get(0).getData());
assertEquals("null", interpreter1.interpret("getProperty property_2", context1).message().get(0).getData());
}
}

5
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java
View file

@ -17,12 +17,11 @@
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import java.io.IOException;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
public class RemoteInterpreterUtilsTest {

12
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/GetEnvPropertyInterpreter.java
View file

@ -16,7 +16,9 @@
*/
package org.apache.zeppelin.interpreter.remote.mock;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@ -25,9 +27,9 @@ import java.util.List;
import java.util.Properties;
public class MockInterpreterEnv extends Interpreter {
public class GetEnvPropertyInterpreter extends Interpreter {
public MockInterpreterEnv(Properties property) {
public GetEnvPropertyInterpreter(Properties property) {
super(property);
}
@ -43,9 +45,9 @@ public class MockInterpreterEnv extends Interpreter {
public InterpreterResult interpret(String st, InterpreterContext context) {
String[] cmd = st.split(" ");
if (cmd[0].equals("getEnv")) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]));
return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]) == null ? "null" : System.getenv(cmd[1]));
} else if (cmd[0].equals("getProperty")){
return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]));
return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]) == null ? "null" : System.getProperty(cmd[1]));
} else {
return new InterpreterResult(InterpreterResult.Code.ERROR, cmd[0]);
}

7
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterA.java
View file

@ -17,19 +17,18 @@
package org.apache.zeppelin.interpreter.remote.mock;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import java.util.List;
import java.util.Properties;
public class MockInterpreterA extends Interpreter {
private String lastSt;

9
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterAngular.java
View file

@ -17,19 +17,18 @@
package org.apache.zeppelin.interpreter.remote.mock;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectWatcher;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
public class MockInterpreterAngular extends Interpreter {
AtomicInteger numWatch = new AtomicInteger(0);

14
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterB.java
View file

@ -17,20 +17,14 @@
package org.apache.zeppelin.interpreter.remote.mock;
import java.util.List;
import java.util.Properties;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import java.util.List;
import java.util.Properties;
public class MockInterpreterB extends Interpreter {
public MockInterpreterB(Properties property) {

5
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterOutputStream.java
View file

@ -16,7 +16,10 @@
*/
package org.apache.zeppelin.interpreter.remote.mock;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;

11
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterResourcePool.java
View file

@ -17,22 +17,19 @@
package org.apache.zeppelin.interpreter.remote.mock;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.gson.Gson;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectWatcher;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.resource.Resource;
import org.apache.zeppelin.resource.ResourcePool;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
public class MockInterpreterResourcePool extends Interpreter {
AtomicInteger numWatch = new AtomicInteger(0);

129
zeppelin-zengine/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java → zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
View file

@ -17,83 +17,86 @@
package org.apache.zeppelin.scheduler;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Properties;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterRunner;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterSetting interpreterSetting;
private SchedulerFactory schedulerSvc;
private static final int TICK_WAIT = 100;
private static final int MAX_WAIT_CYCLES = 100;
@Before
public void setUp() throws Exception{
public void setUp() throws Exception {
schedulerSvc = new SchedulerFactory();
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setRemote(true);
InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterA.class.getName(), "mock", true, new HashMap<String, Object>());
List<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(interpreterInfo1);
InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT);
interpreterSetting = new InterpreterSetting.Builder()
.setId("test")
.setName("test")
.setGroup("test")
.setInterpreterInfos(interpreterInfos)
.setOption(interpreterOption)
.setRunner(runner)
.setInterpreterDir("../interpeters/test")
.create();
}
@After
public void tearDown(){
public void tearDown() {
interpreterSetting.close();
}
@Test
public void test() throws Exception {
Properties p = new Properties();
final InterpreterGroup intpGroup = new InterpreterGroup();
Map<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(
p,
"note",
MockInterpreterA.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
this,
null,
"anonymous",
false);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
intpA.open();
Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
intpA.getInterpreterProcess(),
intpA,
10);
Job job = new Job("jobId", "jobName", null, 200) {
Object results;
@Override
public Object getReturn() {
return results;
@ -120,7 +123,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
return "1000";
@ -145,7 +148,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
}
assertTrue(job.isRunning());
Thread.sleep(5*TICK_WAIT);
Thread.sleep(5 * TICK_WAIT);
assertEquals(0, scheduler.getJobsWaiting().size());
assertEquals(1, scheduler.getJobsRunning().size());
@ -165,34 +168,10 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
@Test
public void testAbortOnPending() throws Exception {
Properties p = new Properties();
final InterpreterGroup intpGroup = new InterpreterGroup();
Map<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
final RemoteInterpreter intpA = new RemoteInterpreter(
p,
"note",
MockInterpreterA.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
this,
null,
"anonymous",
false);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
final RemoteInterpreter intpA = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1");
intpA.open();
Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note",
intpA.getInterpreterProcess(),
10);
Scheduler scheduler = schedulerSvc.createOrGetRemoteScheduler("test", "note", intpA, 10);
Job job1 = new Job("jobId1", "jobName1", null, 200) {
Object results;
@ -205,7 +184,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
@ -255,7 +234,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
null,
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
@ -358,7 +337,7 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener {
}
@Override
public void onParaInfosReceived(String noteId, String paragraphId,
String interpreterSettingId, Map<String, String> metaInfos) {
public void onParaInfosReceived(String noteId, String paragraphId,
String interpreterSettingId, Map<String, String> metaInfos) {
}
}

115
zeppelin-interpreter/src/test/resources/conf/interpreter.json Normal file
View file

@ -0,0 +1,115 @@
{
"interpreterSettings": {
"2C3RWCVAG": {
"id": "2C3RWCVAG",
"name": "test",
"group": "test",
"properties": {
"property_1": "value_1",
"property_2": "new_value_2",
"property_3": "value_3"
},
"status": "READY",
"interpreterGroup": [
{
"name": "echo",
"class": "org.apache.zeppelin.interpreter.EchoInterpreter",
"defaultInterpreter": true,
"editor": {
"language": "java",
"editOnDblClick": false
}
}
],
"dependencies": [],
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
},
"2CKWE7B19": {
"id": "2CKWE7B19",
"name": "test2",
"group": "test",
"properties": {
"property_1": "value_1",
"property_2": "new_value_2",
"property_3": "value_3"
},
"status": "READY",
"interpreterGroup": [
{
"name": "echo",
"class": "org.apache.zeppelin.interpreter.EchoInterpreter",
"defaultInterpreter": true,
"editor": {
"language": "java",
"editOnDblClick": false
}
}
],
"dependencies": [],
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
},
"interpreterBindings": {
"2C6793KRV": [
"2C48Y7FSJ",
"2C63XW4XE",
"2C66GE1VB",
"2C5VH924X",
"2C4BJDRRZ",
"2C3SQSB7V",
"2C4HKDCQW",
"2C3DR183X",
"2C66Z9XPQ",
"2C3PTPMUH",
"2C69WE69N",
"2C5SRRXHM",
"2C4ZD49PF",
"2C6V3D44K",
"2C4UB1UZA",
"2C5S1R21W",
"2C5DCRVGM",
"2C686X8ZH",
"2C3RWCVAG",
"2C3JKFMJU",
"2C3VECEG2"
]
},
"interpreterRepositories": [
{
"id": "central",
"type": "default",
"url": "http://repo1.maven.org/maven2/",
"releasePolicy": {
"enabled": true,
"updatePolicy": "daily",
"checksumPolicy": "warn"
},
"snapshotPolicy": {
"enabled": true,
"updatePolicy": "daily",
"checksumPolicy": "warn"
},
"mirroredRepositories": [],
"repositoryManager": false
}
]
}

42
zeppelin-interpreter/src/test/resources/interpreter/test/interpreter-setting.json Normal file
View file

@ -0,0 +1,42 @@
[
{
"group": "test",
"name": "double_echo",
"className": "org.apache.zeppelin.interpreter.DoubleEchoInterpreter",
"properties": {
"property_1": {
"envName": "PROPERTY_1",
"propertyName": "property_1",
"defaultValue": "value_1",
"description": "desc_1"
},
"property_2": {
"envName": "PROPERTY_2",
"propertyName": "property_2",
"defaultValue": "value_2",
"description": "desc_2"
}
}
},
{
"group": "test",
"name": "echo",
"defaultInterpreter": true,
"className": "org.apache.zeppelin.interpreter.EchoInterpreter",
"properties": {
"property_1": {
"envName": "PROPERTY_1",
"propertyName": "property_1",
"defaultValue": "value_1",
"description": "desc_1"
},
"property_2": {
"envName": "PROPERTY_2",
"propertyName": "property_2",
"defaultValue": "value_2",
"description": "desc_2"
}
}
}
]

4
zeppelin-interpreter/src/test/resources/log4j.properties
View file

@ -26,4 +26,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
#
# Root logger option
log4j.rootLogger=INFO, stdout
log4j.rootLogger=INFO, stdout
log4j.logger.org.apache.zeppelin.interpreter=DEBUG
log4j.logger.org.apache.zeppelin.scheduler=DEBUG

4
zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java
View file

@ -185,7 +185,7 @@ public class InterpreterRestApi {
String noteId = request == null ? null : request.getNoteId();
if (null == noteId) {
interpreterSettingManager.close(setting);
interpreterSettingManager.close(settingId);
} else {
interpreterSettingManager.restart(settingId, noteId, SecurityUtils.getPrincipal());
}
@ -208,7 +208,7 @@ public class InterpreterRestApi {
@GET
@ZeppelinApi
public Response listInterpreter(String message) {
Map<String, InterpreterSetting> m = interpreterSettingManager.getAvailableInterpreterSettings();
Map<String, InterpreterSetting> m = interpreterSettingManager.getInterpreterSettingTemplates();
return new JsonResponse<>(Status.OK, "", m).build();
}

47
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -93,13 +93,11 @@ public class ZeppelinServer extends Application {
private NotebookRepoSync notebookRepo;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private DependencyResolver depResolver;
public ZeppelinServer() throws Exception {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
this.depResolver = new DependencyResolver(
conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO));
InterpreterOutput.limit = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT);
@ -129,27 +127,10 @@ public class ZeppelinServer extends Application {
new File(conf.getRelativeDir("zeppelin-web/src/app/spell")));
}
ZeppelinServer.helium = new Helium(
conf.getHeliumConfPath(),
conf.getHeliumRegistry(),
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
"helium-registry-cache"),
heliumBundleFactory,
heliumApplicationFactory);
// create bundle
try {
heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
this.schedulerFactory = new SchedulerFactory();
this.interpreterSettingManager = new InterpreterSettingManager(conf, depResolver,
new InterpreterOption(true));
this.replFactory = new InterpreterFactory(conf, notebookWsServer,
notebookWsServer, heliumApplicationFactory, depResolver, SecurityUtils.isAuthenticated(),
interpreterSettingManager);
this.schedulerFactory = SchedulerFactory.singleton();
this.interpreterSettingManager = new InterpreterSettingManager(conf, notebookWsServer,
notebookWsServer, notebookWsServer);
this.replFactory = new InterpreterFactory(interpreterSettingManager);
this.notebookRepo = new NotebookRepoSync(conf);
this.noteSearchService = new LuceneSearch();
this.notebookAuthorization = NotebookAuthorization.init(conf);
@ -158,6 +139,22 @@ public class ZeppelinServer extends Application {
notebookRepo, schedulerFactory, replFactory, interpreterSettingManager, notebookWsServer,
noteSearchService, notebookAuthorization, credentials);
ZeppelinServer.helium = new Helium(
conf.getHeliumConfPath(),
conf.getHeliumRegistry(),
new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO),
"helium-registry-cache"),
heliumBundleFactory,
heliumApplicationFactory,
interpreterSettingManager);
// create bundle
try {
heliumBundleFactory.buildAllPackages(helium.getBundlePackagesToBundle());
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
// to update notebook from application event from remote process.
heliumApplicationFactory.setNotebook(notebook);
// to update fire websocket event on application event.
@ -206,7 +203,7 @@ public class ZeppelinServer extends Application {
LOG.info("Shutting down Zeppelin Server ... ");
try {
jettyWebServer.stop();
notebook.getInterpreterSettingManager().shutdown();
notebook.getInterpreterSettingManager().close();
notebook.close();
Thread.sleep(3000);
} catch (Exception e) {

32
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -41,12 +41,7 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.helium.HeliumPackage;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
@ -463,7 +458,8 @@ public class NotebookServer extends WebSocketServlet
Notebook notebook = notebook();
List<Note> notes = notebook.getAllNotes();
for (Note note : notes) {
List<String> ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId());
List<String> ids = notebook.getInterpreterSettingManager()
.getInterpreterBinding(note.getId());
for (String id : ids) {
if (id.equals(interpreterGroupId)) {
broadcast(note.getId(), m);
@ -1003,7 +999,7 @@ public class NotebookServer extends WebSocketServlet
List<String> interpreterSettingIds = new LinkedList<>();
interpreterSettingIds.add(defaultInterpreterId);
for (String interpreterSettingId : notebook.getInterpreterSettingManager().
getDefaultInterpreterSettingList()) {
getInterpreterSettingIds()) {
if (!interpreterSettingId.equals(defaultInterpreterId)) {
interpreterSettingIds.add(interpreterSettingId);
}
@ -1363,12 +1359,13 @@ public class NotebookServer extends WebSocketServlet
List<InterpreterSetting> settings =
notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(user, note.getId()) == null) {
if (setting.getOrCreateInterpreterGroup(user, note.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, note.getId())
.getId())) {
AngularObjectRegistry angularObjectRegistry =
setting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
setting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
// first trying to get local registry
ao = angularObjectRegistry.get(varName, noteId, paragraphId);
@ -1405,12 +1402,13 @@ public class NotebookServer extends WebSocketServlet
List<InterpreterSetting> settings =
notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId());
for (InterpreterSetting setting : settings) {
if (setting.getInterpreterGroup(user, n.getId()) == null) {
if (setting.getOrCreateInterpreterGroup(user, n.getId()) == null) {
continue;
}
if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, n.getId())
.getId())) {
AngularObjectRegistry angularObjectRegistry =
setting.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
setting.getOrCreateInterpreterGroup(user, n.getId()).getAngularObjectRegistry();
this.broadcastExcept(n.getId(),
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
.put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId())
@ -2283,13 +2281,13 @@ public class NotebookServer extends WebSocketServlet
for (InterpreterSetting intpSetting : settings) {
AngularObjectRegistry registry =
intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry();
List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
for (AngularObject object : objects) {
conn.send(serializeMessage(
new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
.put("interpreterGroupId",
intpSetting.getInterpreterGroup(user, note.getId()).getId())
intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getId())
.put("noteId", note.getId()).put("paragraphId", object.getParagraphId())));
}
}
@ -2335,7 +2333,7 @@ public class NotebookServer extends WebSocketServlet
}
List<String> settingIds =
notebook.getInterpreterSettingManager().getInterpreters(note.getId());
notebook.getInterpreterSettingManager().getInterpreterBinding(note.getId());
for (String id : settingIds) {
if (interpreterGroupId.contains(id)) {
broadcast(note.getId(),

7
zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
View file

@ -304,10 +304,9 @@ public abstract class AbstractTestRestApi {
protected static void shutDown() throws Exception {
if (!wasRunning) {
// restart interpreter to stop all interpreter processes
List<String> settingList = ZeppelinServer.notebook.getInterpreterSettingManager()
.getDefaultInterpreterSettingList();
for (String setting : settingList) {
ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting);
List<InterpreterSetting> settingList = ZeppelinServer.notebook.getInterpreterSettingManager().get();
for (InterpreterSetting setting : settingList) {
ZeppelinServer.notebook.getInterpreterSettingManager().restart(setting.getId());
}
if (shiroIni != null) {
FileUtils.deleteQuietly(shiroIni);

6
zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java
View file

@ -80,7 +80,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
// then
assertThat(get, isAllowed());
assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getAvailableInterpreterSettings().size(),
assertEquals(ZeppelinServer.notebook.getInterpreterSettingManager().getInterpreterSettingTemplates().size(),
body.entrySet().size());
get.releaseConnection();
}
@ -110,7 +110,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testSettingsCRUD() throws IOException {
// when: call create setting API
String rawRequest = "{\"name\":\"md2\",\"group\":\"md\"," +
String rawRequest = "{\"name\":\"md3\",\"group\":\"md\"," +
"\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +
@ -367,7 +367,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi {
@Test
public void testGetMetadataInfo() throws IOException {
String jsonRequest = "{\"name\":\"spark\",\"group\":\"spark\"," +
String jsonRequest = "{\"name\":\"spark_new\",\"group\":\"spark\"," +
"\"properties\":{\"propname\": {\"value\": \"propvalue\", \"name\": \"propname\", \"type\": \"textarea\"}}," +
"\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
"\"dependencies\":[]," +

13
zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java
View file

@ -30,6 +30,7 @@ import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.socket.Message;
import org.apache.zeppelin.notebook.socket.Message.OP;
import org.apache.zeppelin.rest.AbstractTestRestApi;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.AfterClass;
@ -95,7 +96,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
}
@Test
public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException {
public void testMakeSureNoAngularObjectBroadcastToWebsocketWhoFireTheEvent() throws IOException, InterruptedException {
// create a notebook
Note note1 = notebook.createNote(anonymous);
@ -104,7 +105,7 @@ public class NotebookServerTest extends AbstractTestRestApi {
List<InterpreterSetting> settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note1.getId());
for (InterpreterSetting setting : settings) {
if (setting.getName().equals("md")) {
interpreterGroup = setting.getInterpreterGroup("anonymous", "sharedProcess");
interpreterGroup = setting.getOrCreateInterpreterGroup("anonymous", "sharedProcess");
break;
}
}
@ -115,6 +116,14 @@ public class NotebookServerTest extends AbstractTestRestApi {
p1.setAuthenticationInfo(anonymous);
note1.run(p1.getId());
// wait for paragraph finished
while(true) {
if (p1.getStatus() == Job.Status.FINISHED) {
break;
}
Thread.sleep(100);
}
// add angularObject
interpreterGroup.getAngularObjectRegistry().add("object1", "value1", note1.getId(), null);

1
zeppelin-server/src/test/resources/log4j.properties
View file

@ -33,7 +33,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
log4j.logger.org.apache.zeppelin.scheduler=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN

13
zeppelin-zengine/pom.xml
View file

@ -60,6 +60,14 @@
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@ -70,11 +78,6 @@
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>

51
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
View file

@ -19,11 +19,12 @@ package org.apache.zeppelin.helium;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.resource.DistributedResourcePool;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.resource.ResourceSet;
import org.apache.zeppelin.resource.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -47,19 +48,22 @@ public class Helium {
private final HeliumBundleFactory bundleFactory;
private final HeliumApplicationFactory applicationFactory;
private final InterpreterSettingManager interpreterSettingManager;
public Helium(
String heliumConfPath,
String registryPaths,
File registryCacheDir,
HeliumBundleFactory bundleFactory,
HeliumApplicationFactory applicationFactory)
HeliumApplicationFactory applicationFactory,
InterpreterSettingManager interpreterSettingManager)
throws IOException {
this.heliumConfPath = heliumConfPath;
this.registryPaths = registryPaths;
this.registryCacheDir = registryCacheDir;
this.bundleFactory = bundleFactory;
this.applicationFactory = applicationFactory;
this.interpreterSettingManager = interpreterSettingManager;
heliumConf = loadConf(heliumConfPath);
allPackages = getAllPackageInfo();
}
@ -350,7 +354,7 @@ public class Helium {
allResources = resourcePool.getAll();
}
} else {
allResources = ResourcePoolUtils.getAllResources();
allResources = interpreterSettingManager.getAllResources();
}
for (List<HeliumPackageSearchResult> pkgs : allPackages.values()) {
@ -478,4 +482,39 @@ public class Helium {
return mixed;
}
public ResourceSet getAllResources() {
return getAllResourcesExcept(null);
}
private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) {
ResourceSet resourceSet = new ResourceSet();
for (InterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
if (interpreterGroupExcludsion != null &&
intpGroup.getId().equals(interpreterGroupExcludsion)) {
continue;
}
RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess();
if (remoteInterpreterProcess == null) {
ResourcePool localPool = intpGroup.getResourcePool();
if (localPool != null) {
resourceSet.addAll(localPool.getAll());
}
} else if (remoteInterpreterProcess.isRunning()) {
List<String> resourceList = remoteInterpreterProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<List<String>>() {
@Override
public List<String> call(RemoteInterpreterService.Client client) throws Exception {
return client.resourcePoolGetAll();
}
}
);
for (String res : resourceList) {
resourceSet.add(Resource.fromJson(res));
}
}
}
return resourceSet;
}
}

118
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java
View file

@ -105,38 +105,33 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
private void load(RemoteInterpreterProcess intpProcess, ApplicationState appState)
throws Exception {
RemoteInterpreterService.Client client = null;
synchronized (appState) {
if (appState.getStatus() == ApplicationState.Status.LOADED) {
// already loaded
return;
}
try {
appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING);
String pkgInfo = pkg.toJson();
String appId = appState.getId();
appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING);
final String pkgInfo = pkg.toJson();
final String appId = appState.getId();
client = intpProcess.getClient();
RemoteApplicationResult ret = client.loadApplication(
appId,
pkgInfo,
paragraph.getNote().getId(),
paragraph.getId());
if (ret.isSuccess()) {
appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED);
} else {
throw new ApplicationException(ret.getMsg());
}
} catch (TException e) {
intpProcess.releaseBrokenClient(client);
throw e;
} finally {
if (client != null) {
intpProcess.releaseClient(client);
}
RemoteApplicationResult ret = intpProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
@Override
public RemoteApplicationResult call(RemoteInterpreterService.Client client)
throws Exception {
return client.loadApplication(
appId,
pkgInfo,
paragraph.getNote().getId(),
paragraph.getId());
}
}
);
if (ret.isSuccess()) {
appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED);
} else {
throw new ApplicationException(ret.getMsg());
}
}
}
@ -199,7 +194,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
}
private void unload(ApplicationState appsToUnload) throws ApplicationException {
private void unload(final ApplicationState appsToUnload) throws ApplicationException {
synchronized (appsToUnload) {
if (appsToUnload.getStatus() != ApplicationState.Status.LOADED) {
throw new ApplicationException(
@ -217,26 +212,19 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
throw new ApplicationException("Target interpreter process is not running");
}
RemoteInterpreterService.Client client;
try {
client = intpProcess.getClient();
} catch (Exception e) {
throw new ApplicationException(e);
}
try {
RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId());
if (ret.isSuccess()) {
appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED);
} else {
throw new ApplicationException(ret.getMsg());
}
} catch (TException e) {
intpProcess.releaseBrokenClient(client);
throw new ApplicationException(e);
} finally {
intpProcess.releaseClient(client);
RemoteApplicationResult ret = intpProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
@Override
public RemoteApplicationResult call(RemoteInterpreterService.Client client)
throws Exception {
return client.unloadApplication(appsToUnload.getId());
}
}
);
if (ret.isSuccess()) {
appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED);
} else {
throw new ApplicationException(ret.getMsg());
}
}
}
@ -286,7 +274,7 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
}
}
private void run(ApplicationState app) throws ApplicationException {
private void run(final ApplicationState app) throws ApplicationException {
synchronized (app) {
if (app.getStatus() != ApplicationState.Status.LOADED) {
throw new ApplicationException(
@ -303,29 +291,19 @@ public class HeliumApplicationFactory implements ApplicationEventListener, Noteb
if (intpProcess == null) {
throw new ApplicationException("Target interpreter process is not running");
}
RemoteInterpreterService.Client client = null;
try {
client = intpProcess.getClient();
} catch (Exception e) {
throw new ApplicationException(e);
}
try {
RemoteApplicationResult ret = client.runApplication(app.getId());
if (ret.isSuccess()) {
// success
} else {
throw new ApplicationException(ret.getMsg());
}
} catch (TException e) {
intpProcess.releaseBrokenClient(client);
client = null;
throw new ApplicationException(e);
} finally {
if (client != null) {
intpProcess.releaseClient(client);
}
RemoteApplicationResult ret = intpProcess.callRemoteFunction(
new RemoteInterpreterProcess.RemoteFunction<RemoteApplicationResult>() {
@Override
public RemoteApplicationResult call(RemoteInterpreterService.Client client)
throws Exception {
return client.runApplication(app.getId());
}
}
);
if (ret.isSuccess()) {
// success
} else {
throw new ApplicationException(ret.getMsg());
}
}
}

423
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
View file

@ -1,423 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.NullArgumentException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Manage interpreters.
*/
public class InterpreterFactory implements InterpreterGroupFactory {
private static final Logger logger = LoggerFactory.getLogger(InterpreterFactory.class);
private Map<String, URLClassLoader> cleanCl =
Collections.synchronizedMap(new HashMap<String, URLClassLoader>());
private ZeppelinConfiguration conf;
private final InterpreterSettingManager interpreterSettingManager;
private AngularObjectRegistryListener angularObjectRegistryListener;
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private final ApplicationEventListener appEventListener;
private boolean shiroEnabled;
private Map<String, String> env = new HashMap<>();
private Interpreter devInterpreter;
public InterpreterFactory(ZeppelinConfiguration conf,
AngularObjectRegistryListener angularObjectRegistryListener,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appEventListener, DependencyResolver depResolver,
boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager)
throws InterpreterException, IOException, RepositoryException {
this.conf = conf;
this.angularObjectRegistryListener = angularObjectRegistryListener;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.appEventListener = appEventListener;
this.shiroEnabled = shiroEnabled;
this.interpreterSettingManager = interpreterSettingManager;
//TODO(jl): Fix it not to use InterpreterGroupFactory
interpreterSettingManager.setInterpreterGroupFactory(this);
logger.info("shiroEnabled: {}", shiroEnabled);
}
/**
* @param id interpreterGroup id. Combination of interpreterSettingId + noteId/userId/shared
* depends on interpreter mode
*/
@Override
public InterpreterGroup createInterpreterGroup(String id, InterpreterOption option)
throws InterpreterException, NullArgumentException {
//When called from REST API without option we receive NPE
if (option == null) {
throw new NullArgumentException("option");
}
AngularObjectRegistry angularObjectRegistry;
InterpreterGroup interpreterGroup = new InterpreterGroup(id);
if (option.isRemote()) {
angularObjectRegistry =
new RemoteAngularObjectRegistry(id, angularObjectRegistryListener, interpreterGroup);
} else {
angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener);
// TODO(moon) : create distributed resource pool for local interpreters and set
}
interpreterGroup.setAngularObjectRegistry(angularObjectRegistry);
return interpreterGroup;
}
public void createInterpretersForNote(InterpreterSetting interpreterSetting, String user,
String noteId, String interpreterSessionKey) {
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
InterpreterOption option = interpreterSetting.getOption();
Properties properties = interpreterSetting.getFlatProperties();
// if interpreters are already there, wait until they're being removed
synchronized (interpreterGroup) {
long interpreterRemovalWaitStart = System.nanoTime();
// interpreter process supposed to be terminated by RemoteInterpreterProcess.dereference()
// in ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT msec. However, if termination of the process and
// removal from interpreter group take too long, throw an error.
long minTimeout = 10L * 1000 * 1000000; // 10 sec
long interpreterRemovalWaitTimeout = Math.max(minTimeout,
conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT) * 1000000L * 2);
while (interpreterGroup.containsKey(interpreterSessionKey)) {
if (System.nanoTime() - interpreterRemovalWaitStart > interpreterRemovalWaitTimeout) {
throw new InterpreterException("Can not create interpreter");
}
try {
interpreterGroup.wait(1000);
} catch (InterruptedException e) {
logger.debug(e.getMessage(), e);
}
}
}
logger.info("Create interpreter instance {} for note {}", interpreterSetting.getName(), noteId);
List<InterpreterInfo> interpreterInfos = interpreterSetting.getInterpreterInfos();
String path = interpreterSetting.getPath();
InterpreterRunner runner = interpreterSetting.getInterpreterRunner();
Interpreter interpreter;
for (InterpreterInfo info : interpreterInfos) {
if (option.isRemote()) {
if (option.isExistingProcess()) {
interpreter =
connectToRemoteRepl(interpreterSessionKey, info.getClassName(), option.getHost(),
option.getPort(), properties, interpreterSetting.getId(), user,
option.isUserImpersonate);
} else {
interpreter = createRemoteRepl(path, interpreterSessionKey, info.getClassName(),
properties, interpreterSetting.getId(), user, option.isUserImpersonate(), runner);
}
} else {
interpreter = createRepl(interpreterSetting.getPath(), info.getClassName(), properties);
}
synchronized (interpreterGroup) {
List<Interpreter> interpreters = interpreterGroup.get(interpreterSessionKey);
if (null == interpreters) {
interpreters = new ArrayList<>();
interpreterGroup.put(interpreterSessionKey, interpreters);
}
if (info.isDefaultInterpreter()) {
interpreters.add(0, interpreter);
} else {
interpreters.add(interpreter);
}
}
logger.info("Interpreter {} {} created", interpreter.getClassName(), interpreter.hashCode());
interpreter.setInterpreterGroup(interpreterGroup);
}
}
private Interpreter createRepl(String dirName, String className, Properties property)
throws InterpreterException {
logger.info("Create repl {} from {}", className, dirName);
ClassLoader oldcl = Thread.currentThread().getContextClassLoader();
try {
URLClassLoader ccl = cleanCl.get(dirName);
if (ccl == null) {
// classloader fallback
ccl = URLClassLoader.newInstance(new URL[]{}, oldcl);
}
boolean separateCL = true;
try { // check if server's classloader has driver already.
Class cls = this.getClass().forName(className);
if (cls != null) {
separateCL = false;
}
} catch (Exception e) {
logger.error("exception checking server classloader driver", e);
}
URLClassLoader cl;
if (separateCL == true) {
cl = URLClassLoader.newInstance(new URL[]{}, ccl);
} else {
cl = ccl;
}
Thread.currentThread().setContextClassLoader(cl);
Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className);
Constructor<Interpreter> constructor =
replClass.getConstructor(new Class[]{Properties.class});
Interpreter repl = constructor.newInstance(property);
repl.setClassloaderUrls(ccl.getURLs());
LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl));
return intp;
} catch (SecurityException e) {
throw new InterpreterException(e);
} catch (NoSuchMethodException e) {
throw new InterpreterException(e);
} catch (IllegalArgumentException e) {
throw new InterpreterException(e);
} catch (InstantiationException e) {
throw new InterpreterException(e);
} catch (IllegalAccessException e) {
throw new InterpreterException(e);
} catch (InvocationTargetException e) {
throw new InterpreterException(e);
} catch (ClassNotFoundException e) {
throw new InterpreterException(e);
} finally {
Thread.currentThread().setContextClassLoader(oldcl);
}
}
private Interpreter connectToRemoteRepl(String interpreterSessionKey, String className,
String host, int port, Properties property, String interpreterSettingId, String userName,
Boolean isUserImpersonate) {
int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
LazyOpenInterpreter intp = new LazyOpenInterpreter(
new RemoteInterpreter(property, interpreterSessionKey, className, host, port, localRepoPath,
connectTimeout, maxPoolSize, remoteInterpreterProcessListener, appEventListener,
userName, isUserImpersonate, conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT)));
return intp;
}
Interpreter createRemoteRepl(String interpreterPath, String interpreterSessionKey,
String className, Properties property, String interpreterSettingId,
String userName, Boolean isUserImpersonate, InterpreterRunner interpreterRunner) {
int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
String interpreterRunnerPath;
String interpreterGroupName = interpreterSettingManager.get(interpreterSettingId).getName();
if (null != interpreterRunner) {
interpreterRunnerPath = interpreterRunner.getPath();
Path p = Paths.get(interpreterRunnerPath);
if (!p.isAbsolute()) {
interpreterRunnerPath = Joiner.on(File.separator)
.join(interpreterPath, interpreterRunnerPath);
}
} else {
interpreterRunnerPath = conf.getInterpreterRemoteRunnerPath();
}
RemoteInterpreter remoteInterpreter =
new RemoteInterpreter(property, interpreterSessionKey, className,
interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize,
remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate,
conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName);
remoteInterpreter.addEnv(env);
return new LazyOpenInterpreter(remoteInterpreter);
}
private List<Interpreter> createOrGetInterpreterList(String user, String noteId,
InterpreterSetting setting) {
InterpreterGroup interpreterGroup = setting.getInterpreterGroup(user, noteId);
synchronized (interpreterGroup) {
String interpreterSessionKey =
interpreterSettingManager.getInterpreterSessionKey(user, noteId, setting);
if (!interpreterGroup.containsKey(interpreterSessionKey)) {
createInterpretersForNote(setting, user, noteId, interpreterSessionKey);
}
return interpreterGroup.get(interpreterSessionKey);
}
}
private InterpreterSetting getInterpreterSettingByGroup(List<InterpreterSetting> settings,
String group) {
Preconditions.checkNotNull(group, "group should be not null");
for (InterpreterSetting setting : settings) {
if (group.equals(setting.getName())) {
return setting;
}
}
return null;
}
private String getInterpreterClassFromInterpreterSetting(InterpreterSetting setting,
String name) {
Preconditions.checkNotNull(name, "name should be not null");
for (InterpreterInfo info : setting.getInterpreterInfos()) {
String infoName = info.getName();
if (null != info.getName() && name.equals(infoName)) {
return info.getClassName();
}
}
return null;
}
private Interpreter getInterpreter(String user, String noteId, InterpreterSetting setting,
String name) {
Preconditions.checkNotNull(noteId, "noteId should be not null");
Preconditions.checkNotNull(setting, "setting should be not null");
Preconditions.checkNotNull(name, "name should be not null");
String className;
if (null != (className = getInterpreterClassFromInterpreterSetting(setting, name))) {
List<Interpreter> interpreterGroup = createOrGetInterpreterList(user, noteId, setting);
for (Interpreter interpreter : interpreterGroup) {
if (className.equals(interpreter.getClassName())) {
return interpreter;
}
}
}
return null;
}
public Interpreter getInterpreter(String user, String noteId, String replName) {
List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(noteId);
InterpreterSetting setting;
Interpreter interpreter;
if (settings == null || settings.size() == 0) {
return null;
}
if (replName == null || replName.trim().length() == 0) {
// get default settings (first available)
// TODO(jl): Fix it in case of returning null
InterpreterSetting defaultSettings = interpreterSettingManager
.getDefaultInterpreterSetting(settings);
return createOrGetInterpreterList(user, noteId, defaultSettings).get(0);
}
String[] replNameSplit = replName.split("\\.");
if (replNameSplit.length == 2) {
String group = null;
String name = null;
group = replNameSplit[0];
name = replNameSplit[1];
setting = getInterpreterSettingByGroup(settings, group);
if (null != setting) {
interpreter = getInterpreter(user, noteId, setting, name);
if (null != interpreter) {
return interpreter;
}
}
throw new InterpreterException(replName + " interpreter not found");
} else {
// first assume replName is 'name' of interpreter. ('groupName' is ommitted)
// search 'name' from first (default) interpreter group
// TODO(jl): Handle with noteId to support defaultInterpreter per note.
setting = interpreterSettingManager.getDefaultInterpreterSetting(settings);
interpreter = getInterpreter(user, noteId, setting, replName);
if (null != interpreter) {
return interpreter;
}
// next, assume replName is 'group' of interpreter ('name' is ommitted)
// search interpreter group and return first interpreter.
setting = getInterpreterSettingByGroup(settings, replName);
if (null != setting) {
List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, setting);
if (null != interpreters) {
return interpreters.get(0);
}
}
// Support the legacy way to use it
for (InterpreterSetting s : settings) {
if (s.getGroup().equals(replName)) {
List<Interpreter> interpreters = createOrGetInterpreterList(user, noteId, s);
if (null != interpreters) {
return interpreters.get(0);
}
}
}
}
return null;
}
public Map<String, String> getEnv() {
return env;
}
public void setEnv(Map<String, String> env) {
this.env = env;
}
}

26
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroupFactory.java
View file

@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import org.apache.commons.lang.NullArgumentException;
/**
* Created InterpreterGroup
*/
public interface InterpreterGroupFactory {
InterpreterGroup createInterpreterGroup(String interpreterGroupId, InterpreterOption option);
}

459
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
View file

@ -1,459 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.zeppelin.dep.Dependency;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.annotations.SerializedName;
import com.google.gson.internal.StringMap;
import static org.apache.zeppelin.notebook.utility.IdHashes.generateId;
/**
* Interpreter settings
*/
public class InterpreterSetting {
private static final Logger logger = LoggerFactory.getLogger(InterpreterSetting.class);
private static final String SHARED_PROCESS = "shared_process";
private String id;
private String name;
// always be null in case of InterpreterSettingRef
private String group;
private transient Map<String, String> infos;
// Map of the note and paragraphs which has runtime infos generated by this interpreter setting.
// This map is used to clear the infos in paragraph when the interpretersetting is restarted
private transient Map<String, Set<String>> runtimeInfosToBeCleared;
/**
* properties can be either Map<String, DefaultInterpreterProperty> or
* Map<String, InterpreterProperty>
* properties should be:
* - Map<String, InterpreterProperty> when Interpreter instances are saved to
* `conf/interpreter.json` file
* - Map<String, DefaultInterpreterProperty> when Interpreters are registered
* : this is needed after https://github.com/apache/zeppelin/pull/1145
* which changed the way of getting default interpreter setting AKA interpreterSettingsRef
*/
private Object properties;
private Status status;
private String errorReason;
@SerializedName("interpreterGroup")
private List<InterpreterInfo> interpreterInfos;
private final transient Map<String, InterpreterGroup> interpreterGroupRef = new HashMap<>();
private List<Dependency> dependencies = new LinkedList<>();
private InterpreterOption option;
private transient String path;
@SerializedName("runner")
private InterpreterRunner interpreterRunner;
@Deprecated
private transient InterpreterGroupFactory interpreterGroupFactory;
private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock;
private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock;
public InterpreterSetting() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
interpreterGroupReadLock = lock.readLock();
interpreterGroupWriteLock = lock.writeLock();
}
public InterpreterSetting(String id, String name, String group,
List<InterpreterInfo> interpreterInfos, Object properties, List<Dependency> dependencies,
InterpreterOption option, String path, InterpreterRunner runner) {
this();
this.id = id;
this.name = name;
this.group = group;
this.interpreterInfos = interpreterInfos;
this.properties = properties;
this.dependencies = dependencies;
this.option = option;
this.path = path;
this.status = Status.READY;
this.interpreterRunner = runner;
}
public InterpreterSetting(String name, String group, List<InterpreterInfo> interpreterInfos,
Object properties, List<Dependency> dependencies, InterpreterOption option, String path,
InterpreterRunner runner) {
this(generateId(), name, group, interpreterInfos, properties, dependencies, option, path,
runner);
}
/**
* Create interpreter from interpreterSettingRef
*
* @param o interpreterSetting from interpreterSettingRef
*/
public InterpreterSetting(InterpreterSetting o) {
this(generateId(), o.getName(), o.getGroup(), o.getInterpreterInfos(), o.getProperties(),
o.getDependencies(), o.getOption(), o.getPath(), o.getInterpreterRunner());
}
public String getId() {
return id;
}
public String getName() {
return name;
}
public String getGroup() {
return group;
}
private String getInterpreterProcessKey(String user, String noteId) {
InterpreterOption option = getOption();
String key;
if (getOption().isExistingProcess) {
key = Constants.EXISTING_PROCESS;
} else if (getOption().isProcess()) {
key = (option.perUserIsolated() ? user : "") + ":" + (option.perNoteIsolated() ? noteId : "");
} else {
key = SHARED_PROCESS;
}
//logger.debug("getInterpreterProcessKey: {} for InterpreterSetting Id: {}, Name: {}",
// key, getId(), getName());
return key;
}
private boolean isEqualInterpreterKeyProcessKey(String refKey, String processKey) {
InterpreterOption option = getOption();
int validCount = 0;
if (getOption().isProcess()
&& !(option.perUserIsolated() == true && option.perNoteIsolated() == true)) {
List<String> processList = Arrays.asList(processKey.split(":"));
List<String> refList = Arrays.asList(refKey.split(":"));
if (refList.size() <= 1 || processList.size() <= 1) {
return refKey.equals(processKey);
}
if (processList.get(0).equals("") || processList.get(0).equals(refList.get(0))) {
validCount = validCount + 1;
}
if (processList.get(1).equals("") || processList.get(1).equals(refList.get(1))) {
validCount = validCount + 1;
}
return (validCount >= 2);
} else {
return refKey.equals(processKey);
}
}
String getInterpreterSessionKey(String user, String noteId) {
InterpreterOption option = getOption();
String key;
if (option.isExistingProcess()) {
key = Constants.EXISTING_PROCESS;
} else if (option.perNoteScoped() && option.perUserScoped()) {
key = user + ":" + noteId;
} else if (option.perUserScoped()) {
key = user;
} else if (option.perNoteScoped()) {
key = noteId;
} else {
key = "shared_session";
}
logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " +
"{}", key, noteId, user, getName());
return key;
}
public InterpreterGroup getInterpreterGroup(String user, String noteId) {
String key = getInterpreterProcessKey(user, noteId);
if (!interpreterGroupRef.containsKey(key)) {
String interpreterGroupId = getId() + ":" + key;
InterpreterGroup intpGroup =
interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption());
interpreterGroupWriteLock.lock();
logger.debug("create interpreter group with groupId:" + interpreterGroupId);
interpreterGroupRef.put(key, intpGroup);
interpreterGroupWriteLock.unlock();
}
try {
interpreterGroupReadLock.lock();
return interpreterGroupRef.get(key);
} finally {
interpreterGroupReadLock.unlock();
}
}
public Collection<InterpreterGroup> getAllInterpreterGroups() {
try {
interpreterGroupReadLock.lock();
return new LinkedList<>(interpreterGroupRef.values());
} finally {
interpreterGroupReadLock.unlock();
}
}
void closeAndRemoveInterpreterGroup(String noteId, String user) {
if (user.equals("anonymous")) {
user = "";
}
String processKey = getInterpreterProcessKey(user, noteId);
String sessionKey = getInterpreterSessionKey(user, noteId);
List<InterpreterGroup> groupToRemove = new LinkedList<>();
InterpreterGroup groupItem;
for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
if (isEqualInterpreterKeyProcessKey(intpKey, processKey)) {
interpreterGroupWriteLock.lock();
// TODO(jl): interpreterGroup has two or more sessionKeys inside it. thus we should not
// remove interpreterGroup if it has two or more values.
groupItem = interpreterGroupRef.get(intpKey);
interpreterGroupWriteLock.unlock();
groupToRemove.add(groupItem);
}
for (InterpreterGroup groupToClose : groupToRemove) {
// TODO(jl): Fix the logic removing session. Now, it's handled into groupToClose.clsose()
groupToClose.close(interpreterGroupRef, intpKey, sessionKey);
}
groupToRemove.clear();
}
//Remove session because all interpreters in this session are closed
//TODO(jl): Change all code to handle interpreter one by one or all at once
}
void closeAndRemoveAllInterpreterGroups() {
for (String processKey : new HashSet<>(interpreterGroupRef.keySet())) {
InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
for (String sessionKey : new HashSet<>(interpreterGroup.keySet())) {
interpreterGroup.close(interpreterGroupRef, processKey, sessionKey);
}
}
}
void shutdownAndRemoveAllInterpreterGroups() {
for (InterpreterGroup interpreterGroup : interpreterGroupRef.values()) {
interpreterGroup.shutdown();
}
}
public Object getProperties() {
return properties;
}
public Properties getFlatProperties() {
Properties p = new Properties();
if (properties != null) {
Map<String, InterpreterProperty> propertyMap = (Map<String, InterpreterProperty>) properties;
for (String key : propertyMap.keySet()) {
InterpreterProperty tmp = propertyMap.get(key);
p.put(tmp.getName() != null ? tmp.getName() : key,
tmp.getValue() != null ? tmp.getValue().toString() : null);
}
}
return p;
}
public List<Dependency> getDependencies() {
if (dependencies == null) {
return new LinkedList<>();
}
return dependencies;
}
public void setDependencies(List<Dependency> dependencies) {
this.dependencies = dependencies;
}
public InterpreterOption getOption() {
if (option == null) {
option = new InterpreterOption();
}
return option;
}
public void setOption(InterpreterOption option) {
this.option = option;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
public List<InterpreterInfo> getInterpreterInfos() {
return interpreterInfos;
}
void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) {
this.interpreterGroupFactory = interpreterGroupFactory;
}
void appendDependencies(List<Dependency> dependencies) {
for (Dependency dependency : dependencies) {
if (!this.dependencies.contains(dependency)) {
this.dependencies.add(dependency);
}
}
}
void setInterpreterOption(InterpreterOption interpreterOption) {
this.option = interpreterOption;
}
public void setProperties(Map<String, InterpreterProperty> p) {
this.properties = p;
}
void setGroup(String group) {
this.group = group;
}
void setName(String name) {
this.name = name;
}
/***
* Interpreter status
*/
public enum Status {
DOWNLOADING_DEPENDENCIES,
ERROR,
READY
}
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
public String getErrorReason() {
return errorReason;
}
public void setErrorReason(String errorReason) {
this.errorReason = errorReason;
}
public void setInfos(Map<String, String> infos) {
this.infos = infos;
}
public Map<String, String> getInfos() {
return infos;
}
public InterpreterRunner getInterpreterRunner() {
return interpreterRunner;
}
public void setInterpreterRunner(InterpreterRunner interpreterRunner) {
this.interpreterRunner = interpreterRunner;
}
public void addNoteToPara(String noteId, String paraId) {
if (runtimeInfosToBeCleared == null) {
runtimeInfosToBeCleared = new HashMap<>();
}
Set<String> paraIdSet = runtimeInfosToBeCleared.get(noteId);
if (paraIdSet == null) {
paraIdSet = new HashSet<>();
runtimeInfosToBeCleared.put(noteId, paraIdSet);
}
paraIdSet.add(paraId);
}
public Map<String, Set<String>> getNoteIdAndParaMap() {
return runtimeInfosToBeCleared;
}
public void clearNoteIdAndParaMap() {
runtimeInfosToBeCleared = null;
}
// For backward compatibility of interpreter.json format after ZEPPELIN-2654
public void convertPermissionsFromUsersToOwners(JsonObject jsonObject) {
if (jsonObject != null) {
JsonObject option = jsonObject.getAsJsonObject("option");
if (option != null) {
JsonArray users = option.getAsJsonArray("users");
if (users != null) {
if (this.option.getOwners() == null) {
this.option.owners = new LinkedList<>();
}
for (JsonElement user : users) {
this.option.getOwners().add(user.getAsString());
}
}
}
}
}
// For backward compatibility of interpreter.json format after ZEPPELIN-2403
public void convertFlatPropertiesToPropertiesWithWidgets() {
StringMap newProperties = new StringMap();
if (properties != null && properties instanceof StringMap) {
StringMap p = (StringMap) properties;
for (Object o : p.entrySet()) {
Map.Entry entry = (Map.Entry) o;
if (!(entry.getValue() instanceof StringMap)) {
StringMap newProperty = new StringMap();
newProperty.put("name", entry.getKey());
newProperty.put("value", entry.getValue());
newProperty.put("type", InterpreterPropertyType.TEXTAREA.getValue());
newProperties.put(entry.getKey().toString(), newProperty);
} else {
// already converted
return;
}
}
this.properties = newProperties;
}
}
}

1136
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
View file

File diff suppressed because it is too large Load diff

597
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
View file

@ -1,597 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import java.util.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.thrift.TException;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
/**
* Proxy for Interpreter instance that runs on separate process
*/
public class RemoteInterpreter extends Interpreter {
private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class);
private final RemoteInterpreterProcessListener remoteInterpreterProcessListener;
private final ApplicationEventListener applicationEventListener;
private Gson gson = new Gson();
private String interpreterRunner;
private String interpreterPath;
private String localRepoPath;
private String className;
private String sessionKey;
private FormType formType;
private boolean initialized;
private Map<String, String> env;
private int connectTimeout;
private int maxPoolSize;
private String host;
private int port;
private String userName;
private Boolean isUserImpersonate;
private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
private String interpreterGroupName;
/**
* Remote interpreter and manage interpreter process
*/
public RemoteInterpreter(Properties property, String sessionKey, String className,
String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout,
int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
int outputLimit, String interpreterGroupName) {
super(property);
this.sessionKey = sessionKey;
this.className = className;
initialized = false;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.localRepoPath = localRepoPath;
env = getEnvFromInterpreterProperty(property);
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.applicationEventListener = appListener;
this.userName = userName;
this.isUserImpersonate = isUserImpersonate;
this.outputLimit = outputLimit;
this.interpreterGroupName = interpreterGroupName;
}
/**
* Connect to existing process
*/
public RemoteInterpreter(Properties property, String sessionKey, String className, String host,
int port, String localRepoPath, int connectTimeout, int maxPoolSize,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
int outputLimit) {
super(property);
this.sessionKey = sessionKey;
this.className = className;
initialized = false;
this.host = host;
this.port = port;
this.localRepoPath = localRepoPath;
this.connectTimeout = connectTimeout;
this.maxPoolSize = maxPoolSize;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.applicationEventListener = appListener;
this.userName = userName;
this.isUserImpersonate = isUserImpersonate;
this.outputLimit = outputLimit;
}
// VisibleForTesting
public RemoteInterpreter(Properties property, String sessionKey, String className,
String interpreterRunner, String interpreterPath, String localRepoPath,
Map<String, String> env, int connectTimeout,
RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate) {
super(property);
this.className = className;
this.sessionKey = sessionKey;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.localRepoPath = localRepoPath;
env.putAll(getEnvFromInterpreterProperty(property));
this.env = env;
this.connectTimeout = connectTimeout;
this.maxPoolSize = 10;
this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
this.applicationEventListener = appListener;
this.userName = userName;
this.isUserImpersonate = isUserImpersonate;
}
private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
Map<String, String> env = new HashMap<String, String>();
StringBuilder sparkConfBuilder = new StringBuilder();
for (String key : property.stringPropertyNames()) {
if (RemoteInterpreterUtils.isEnvString(key)) {
env.put(key, property.getProperty(key));
}
if (key.equals("master")) {
sparkConfBuilder.append(" --master " + property.getProperty("master"));
}
if (isSparkConf(key, property.getProperty(key))) {
sparkConfBuilder.append(" --conf " + key + "=" +
toShellFormat(property.getProperty(key)));
}
}
env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());
return env;
}
private String toShellFormat(String value) {
if (value.contains("\'") && value.contains("\"")) {
throw new RuntimeException("Spark property value could not contain both \" and '");
} else if (value.contains("\'")) {
return "\"" + value + "\"";
} else {
return "\'" + value + "\'";
}
}
static boolean isSparkConf(String key, String value) {
return !StringUtils.isEmpty(key) && key.startsWith("spark.") && !StringUtils.isEmpty(value);
}
@Override
public String getClassName() {
return className;
}
private boolean connectToExistingProcess() {
return host != null && port > 0;
}
public RemoteInterpreterProcess getInterpreterProcess() {
InterpreterGroup intpGroup = getInterpreterGroup();
if (intpGroup == null) {
return null;
}
synchronized (intpGroup) {
if (intpGroup.getRemoteInterpreterProcess() == null) {
RemoteInterpreterProcess remoteProcess;
if (connectToExistingProcess()) {
remoteProcess = new RemoteInterpreterRunningProcess(
connectTimeout,
remoteInterpreterProcessListener,
applicationEventListener,
host,
port);
} else {
// create new remote process
remoteProcess = new RemoteInterpreterManagedProcess(
interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName);
}
intpGroup.setRemoteInterpreterProcess(remoteProcess);
}
return intpGroup.getRemoteInterpreterProcess();
}
}
public synchronized void init() {
if (initialized == true) {
return;
}
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
final InterpreterGroup interpreterGroup = getInterpreterGroup();
interpreterProcess.setMaxPoolSize(
Math.max(this.maxPoolSize, interpreterProcess.getMaxPoolSize()));
String groupId = interpreterGroup.getId();
synchronized (interpreterProcess) {
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
logger.info("Create remote interpreter {}", getClassName());
if (localRepoPath != null) {
property.put("zeppelin.interpreter.localRepo", localRepoPath);
}
property.put("zeppelin.interpreter.output.limit", Integer.toString(outputLimit));
client.createInterpreter(groupId, sessionKey,
getClassName(), (Map) property, userName);
// Push angular object loaded from JSON file to remote interpreter
if (!interpreterGroup.isAngularRegistryPushed()) {
pushAngularObjectRegistryToRemote(client);
interpreterGroup.setAngularRegistryPushed(true);
}
} catch (TException e) {
logger.error("Failed to create interpreter: {}", getClassName());
throw new InterpreterException(e);
} finally {
// TODO(jongyoul): Fixed it when not all of interpreter in same interpreter group are broken
interpreterProcess.releaseClient(client, broken);
}
}
initialized = true;
}
@Override
public void open() {
InterpreterGroup interpreterGroup = getInterpreterGroup();
synchronized (interpreterGroup) {
// initialize all interpreters in this interpreter group
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
// TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
// initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
// doesn't call open method, it's not open. It causes problem while running intp.close()
// In case of Spark, this method initializes all of interpreters and init() method increases
// reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
// other interpreters doesn't do anything because those LazyInterpreters aren't open.
// But for now, we have to initialise all of interpreters for some reasons.
// See Interpreter.getInterpreterInTheSameSessionByClassName(String)
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
if (!initialized) {
// reference per session
interpreterProcess.reference(interpreterGroup, userName, isUserImpersonate);
}
for (Interpreter intp : new ArrayList<>(interpreters)) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
try {
((RemoteInterpreter) p).init();
} catch (InterpreterException e) {
logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
p.getClassName());
interpreters.remove(p);
}
}
}
}
@Override
public void close() {
InterpreterGroup interpreterGroup = getInterpreterGroup();
synchronized (interpreterGroup) {
// close all interpreters in this session
List<Interpreter> interpreters = interpreterGroup.get(sessionKey);
// TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however,
// initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it
// doesn't call open method, it's not open. It causes problem while running intp.close()
// In case of Spark, this method initializes all of interpreters and init() method increases
// reference count of RemoteInterpreterProcess. But while closing this interpreter group, all
// other interpreters doesn't do anything because those LazyInterpreters aren't open.
// But for now, we have to initialise all of interpreters for some reasons.
// See Interpreter.getInterpreterInTheSameSessionByClassName(String)
if (initialized) {
// dereference per session
getInterpreterProcess().dereference();
}
for (Interpreter intp : new ArrayList<>(interpreters)) {
Interpreter p = intp;
while (p instanceof WrappedInterpreter) {
p = ((WrappedInterpreter) p).getInnerInterpreter();
}
try {
((RemoteInterpreter) p).closeInterpreter();
} catch (InterpreterException e) {
logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup",
p.getClassName());
interpreters.remove(p);
}
}
}
}
public void closeInterpreter() {
if (this.initialized == false) {
return;
}
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
boolean broken = false;
try {
client = interpreterProcess.getClient();
if (client != null) {
client.close(sessionKey, className);
}
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} catch (Exception e1) {
throw new InterpreterException(e1);
} finally {
if (client != null) {
interpreterProcess.releaseClient(client, broken);
}
this.initialized = false;
}
}
@Override
public InterpreterResult interpret(String st, InterpreterContext context) {
if (logger.isDebugEnabled()) {
logger.debug("st:\n{}", st);
}
FormType form = getFormType();
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
InterpreterContextRunnerPool interpreterContextRunnerPool = interpreterProcess
.getInterpreterContextRunnerPool();
List<InterpreterContextRunner> runners = context.getRunners();
if (runners != null && runners.size() != 0) {
// assume all runners in this InterpreterContext have the same note id
String noteId = runners.get(0).getNoteId();
interpreterContextRunnerPool.clear(noteId);
interpreterContextRunnerPool.addAll(noteId, runners);
}
boolean broken = false;
try {
final GUI currentGUI = context.getGui();
RemoteInterpreterResult remoteResult = client.interpret(
sessionKey, className, st, convert(context));
Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson(
remoteResult.getConfig(), new TypeToken<Map<String, Object>>() {
}.getType());
context.getConfig().clear();
context.getConfig().putAll(remoteConfig);
if (form == FormType.NATIVE) {
GUI remoteGui = GUI.fromJson(remoteResult.getGui());
currentGUI.clear();
currentGUI.setParams(remoteGui.getParams());
currentGUI.setForms(remoteGui.getForms());
} else if (form == FormType.SIMPLE) {
final Map<String, Input> currentForms = currentGUI.getForms();
final Map<String, Object> currentParams = currentGUI.getParams();
final GUI remoteGUI = GUI.fromJson(remoteResult.getGui());
final Map<String, Input> remoteForms = remoteGUI.getForms();
final Map<String, Object> remoteParams = remoteGUI.getParams();
currentForms.putAll(remoteForms);
currentParams.putAll(remoteParams);
}
InterpreterResult result = convert(remoteResult);
return result;
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
}
@Override
public void cancel(InterpreterContext context) {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
client.cancel(sessionKey, className, convert(context));
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
}
@Override
public FormType getFormType() {
open();
if (formType != null) {
return formType;
}
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
formType = FormType.valueOf(client.getFormType(sessionKey, className));
return formType;
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
}
@Override
public int getProgress(InterpreterContext context) {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
if (interpreterProcess == null || !interpreterProcess.isRunning()) {
return 0;
}
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
return client.getProgress(sessionKey, className, convert(context));
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
Client client = null;
try {
client = interpreterProcess.getClient();
} catch (Exception e1) {
throw new InterpreterException(e1);
}
boolean broken = false;
try {
List completion = client.completion(sessionKey, className, buf, cursor,
convert(interpreterContext));
return completion;
} catch (TException e) {
broken = true;
throw new InterpreterException(e);
} finally {
interpreterProcess.releaseClient(client, broken);
}
}
@Override
public Scheduler getScheduler() {
int maxConcurrency = maxPoolSize;
RemoteInterpreterProcess interpreterProcess = getInterpreterProcess();
if (interpreterProcess == null) {
return null;
} else {
return SchedulerFactory.singleton().createOrGetRemoteScheduler(
RemoteInterpreter.class.getName() + sessionKey + interpreterProcess.hashCode(),
sessionKey, interpreterProcess, maxConcurrency);
}
}
private String getInterpreterGroupKey(InterpreterGroup interpreterGroup) {
return interpreterGroup.getId();
}
private RemoteInterpreterContext convert(InterpreterContext ic) {
return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(),
ic.getParagraphTitle(), ic.getParagraphText(), ic.getAuthenticationInfo().toJson(),
gson.toJson(ic.getConfig()), ic.getGui().toJson(), gson.toJson(ic.getRunners()));
}
private InterpreterResult convert(RemoteInterpreterResult result) {
InterpreterResult r = new InterpreterResult(
InterpreterResult.Code.valueOf(result.getCode()));
for (RemoteInterpreterResultMessage m : result.getMsg()) {
r.add(InterpreterResult.Type.valueOf(m.getType()), m.getData());
}
return r;
}
/**
* Push local angular object registry to
* remote interpreter. This method should be
* call ONLY inside the init() method
*/
void pushAngularObjectRegistryToRemote(Client client) throws TException {
final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup()
.getAngularObjectRegistry();
if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) {
final Map<String, Map<String, AngularObject>> registry = angularObjectRegistry
.getRegistry();
logger.info("Push local angular object registry from ZeppelinServer to" +
" remote interpreter group {}", this.getInterpreterGroup().getId());
final java.lang.reflect.Type registryType = new TypeToken<Map<String,
Map<String, AngularObject>>>() {
}.getType();
Gson gson = new Gson();
client.angularRegistryPush(gson.toJson(registry, registryType));
}
}
public Map<String, String> getEnv() {
return env;
}
public void addEnv(Map<String, String> env) {
if (this.env == null) {
this.env = new HashMap<>();
}
this.env.putAll(env);
}
//Only for test
public String getInterpreterRunner() {
return interpreterRunner;
}
}

1
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java
View file

@ -17,7 +17,6 @@
package org.apache.zeppelin.notebook;
import org.apache.zeppelin.helium.HeliumPackage;
import org.apache.zeppelin.interpreter.InterpreterGroup;
/**
* Current state of application

12
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -41,7 +41,6 @@ import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.utility.IdHashes;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.search.SearchService;
@ -126,11 +125,6 @@ public class Note implements ParagraphJobListener, JsonSerializable {
id = IdHashes.generateId();
}
private String getDefaultInterpreterName() {
InterpreterSetting setting = interpreterSettingManager.getDefaultInterpreterSetting(getId());
return null != setting ? setting.getName() : StringUtils.EMPTY;
}
public boolean isPersonalizedMode() {
Object v = getConfig().get("personalizedMode");
return null != v && "true".equals(v);
@ -385,7 +379,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
*/
public Paragraph removeParagraph(String user, String paragraphId) {
removeAllAngularObjectInParagraph(user, paragraphId);
ResourcePoolUtils.removeResourcesBelongsToParagraph(getId(), paragraphId);
interpreterSettingManager.removeResourcesBelongsToParagraph(getId(), paragraphId);
synchronized (paragraphs) {
Iterator<Paragraph> i = paragraphs.iterator();
while (i.hasNext()) {
@ -690,7 +684,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
angularObjects.put(intpGroup.getId(), registry.getAllWithGlobal(id));
}
@ -705,7 +699,7 @@ public class Note implements ParagraphJobListener, JsonSerializable {
}
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(user, id);
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {

23
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -60,7 +60,6 @@ import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.NotebookRepo.Revision;
import org.apache.zeppelin.notebook.repo.NotebookRepoSync;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
@ -140,7 +139,7 @@ public class Notebook implements NoteEventListener {
Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null");
Note note;
if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) {
note = createNote(interpreterSettingManager.getDefaultInterpreterSettingList(), subject);
note = createNote(interpreterSettingManager.getInterpreterSettingIds(), subject);
} else {
note = createNote(null, subject);
}
@ -270,8 +269,8 @@ public class Notebook implements NoteEventListener {
}
}
interpreterSettingManager.setInterpreters(user, note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreters(...) do the same
interpreterSettingManager.setInterpreterBinding(user, note.getId(), interpreterSettingIds);
// comment out while note.getNoteReplLoader().setInterpreterBinding(...) do the same
// replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds);
}
}
@ -279,7 +278,7 @@ public class Notebook implements NoteEventListener {
List<String> getBindedInterpreterSettingsIds(String id) {
Note note = getNote(id);
if (note != null) {
return interpreterSettingManager.getInterpreters(note.getId());
return interpreterSettingManager.getInterpreterBinding(note.getId());
} else {
return new LinkedList<>();
}
@ -313,9 +312,10 @@ public class Notebook implements NoteEventListener {
}
public void moveNoteToTrash(String noteId) {
for (InterpreterSetting interpreterSetting : interpreterSettingManager
.getInterpreterSettings(noteId)) {
interpreterSettingManager.removeInterpretersForNote(interpreterSetting, "", noteId);
try {
interpreterSettingManager.setInterpreterBinding("", noteId, new ArrayList<String>());
} catch (IOException e) {
e.printStackTrace();
}
}
@ -339,7 +339,7 @@ public class Notebook implements NoteEventListener {
// remove from all interpreter instance's angular object registry
for (InterpreterSetting settings : interpreterSettingManager.get()) {
AngularObjectRegistry registry =
settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
settings.getOrCreateInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry();
if (registry instanceof RemoteAngularObjectRegistry) {
// remove paragraph scope object
for (Paragraph p : note.getParagraphs()) {
@ -374,7 +374,7 @@ public class Notebook implements NoteEventListener {
}
}
ResourcePoolUtils.removeResourcesBelongsToNote(id);
interpreterSettingManager.removeResourcesBelongsToNote(id);
fireNoteRemoveEvent(note);
@ -521,7 +521,8 @@ public class Notebook implements NoteEventListener {
SnapshotAngularObject snapshot = angularObjectSnapshot.get(name);
List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId());
InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(subject.getUser(),
note.getId());
if (intpGroup.getId().equals(snapshot.getIntpGroupId())) {
AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry();
String noteId = snapshot.getAngularObject().getNoteId();

21
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java
View file

@ -93,10 +93,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
// since zeppelin-0.7.0, zeppelin stores multiple results of the paragraph
// see ZEPPELIN-212
Object results;
volatile Object results;
// For backward compatibility of note.json format after ZEPPELIN-212
Object result;
volatile Object result;
private Map<String, ParagraphRuntimeInfo> runtimeInfos;
/**
@ -157,7 +157,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
@Override
public void setResult(Object results) {
public synchronized void setResult(Object results) {
this.results = results;
}
@ -354,7 +354,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
}
@Override
public Object getReturn() {
public synchronized Object getReturn() {
return results;
}
@ -401,6 +401,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
logger.error("Can not find interpreter name " + repl);
throw new RuntimeException("Can not find interpreter for " + getRequiredReplName());
}
//TODO(zjffdu) check interpreter setting status in interpreter setting itself
InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId());
while (intp.getStatus().equals(
org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) {
@ -560,8 +561,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup =
interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
.getAngularObjectRegistry();
resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
.getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();
@ -591,8 +594,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable {
if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) {
InterpreterSetting intpGroup =
interpreterSettingManager.getInterpreterSettings(note.getId()).get(0);
registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry();
resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool();
registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
.getAngularObjectRegistry();
resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId())
.getResourcePool();
}
List<InterpreterContextRunner> runners = new LinkedList<>();

79
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java
View file

@ -16,14 +16,14 @@
*/
package org.apache.zeppelin.helium;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.scheduler.Job;
@ -45,14 +45,9 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
public class HeliumApplicationFactoryTest implements JobListenerFactory {
private File tmpDir;
private File notebookDir;
private ZeppelinConfiguration conf;
public class HeliumApplicationFactoryTest extends AbstractInterpreterTest implements JobListenerFactory {
private SchedulerFactory schedulerFactory;
private DependencyResolver depResolver;
private InterpreterFactory factory;
private InterpreterSettingManager interpreterSettingManager;
private VFSNotebookRepo notebookRepo;
private Notebook notebook;
private HeliumApplicationFactory heliumAppFactory;
@ -60,46 +55,15 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZepelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
File confDir = new File(tmpDir, "conf");
confDir.mkdirs();
notebookDir = new File(tmpDir + "/notebook");
notebookDir.mkdirs();
File home = new File(getClass().getClassLoader().getResource("note").getFile()) // zeppelin/zeppelin-zengine/target/test-classes/note
.getParentFile() // zeppelin/zeppelin-zengine/target/test-classes
.getParentFile() // zeppelin/zeppelin-zengine/target
.getParentFile() // zeppelin/zeppelin-zengine
.getParentFile(); // zeppelin
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), home.getAbsolutePath());
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.getAbsolutePath() + "/conf");
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
conf = new ZeppelinConfiguration();
this.schedulerFactory = new SchedulerFactory();
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
super.setUp();
this.schedulerFactory = SchedulerFactory.singleton();
heliumAppFactory = new HeliumApplicationFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager);
HashMap<String, String> env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
factory.setEnv(env);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
// set AppEventListener properly
for (InterpreterSetting interpreterSetting : interpreterSettingManager.get()) {
interpreterSetting.setAppEventListener(heliumAppFactory);
}
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
@ -108,7 +72,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
conf,
notebookRepo,
schedulerFactory,
factory,
interpreterFactory,
interpreterSettingManager,
this,
search,
@ -124,16 +88,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
@After
public void tearDown() throws Exception {
List<InterpreterSetting> settings = interpreterSettingManager.get();
for (InterpreterSetting setting : settings) {
for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) {
intpGroup.close();
}
}
FileUtils.deleteDirectory(tmpDir);
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(),
ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getStringValue());
super.tearDown();
}
@ -150,7 +105,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note1.getId(),interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note1.getId(),interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -196,7 +151,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -236,7 +191,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -297,7 +252,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory {
"", "");
Note note1 = notebook.createNote(anonymous);
notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getInterpreterSettingIds());
String mock1IntpSettingId = null;
for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) {
if (setting.getName().equals("mock1")) {

8
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java
View file

@ -52,7 +52,7 @@ public class HeliumTest {
// given
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(),
null, null, null);
null, null, null, null);
assertFalse(heliumConf.exists());
// when
@ -63,14 +63,14 @@ public class HeliumTest {
// then load without exception
Helium heliumRestored = new Helium(
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
}
@Test
public void testRestoreRegistryInstances() throws IOException, URISyntaxException, TaskRunnerException {
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1");
HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", "r2");
helium.addRegistry(registry1);
@ -105,7 +105,7 @@ public class HeliumTest {
public void testRefresh() throws IOException, URISyntaxException, TaskRunnerException {
File heliumConf = new File(tmpDir, "helium.conf");
Helium helium = new Helium(
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null);
heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath(), null, null, null, null);
HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1");
helium.addRegistry(registry1);

497
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
View file

@ -1,497 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.NullArgumentException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mock;
import org.quartz.SchedulerException;
import org.sonatype.aether.RepositoryException;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class InterpreterFactoryTest {
private InterpreterFactory factory;
private InterpreterSettingManager interpreterSettingManager;
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterContext context;
private Notebook notebook;
private NotebookRepo notebookRepo;
private DependencyResolver depResolver;
private SchedulerFactory schedulerFactory;
private NotebookAuthorization notebookAuthorization;
@Mock
private JobListenerFactory jobListenerFactory;
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
new File(tmpDir, "conf").mkdirs();
FileUtils.copyDirectory(new File("src/test/resources/interpreter"), new File(tmpDir, "interpreter"));
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(),
"mock1,mock2,mock11,dev");
conf = new ZeppelinConfiguration();
schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
intp1Properties.put("PROPERTY_1",
new InterpreterProperty("PROPERTY_1", "VALUE_1"));
intp1Properties.put("property_2",
new InterpreterProperty("property_2", "value_2"));
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, jobListenerFactory, search,
notebookAuthorization, null);
}
@After
public void tearDown() throws Exception {
FileUtils.deleteDirectory(tmpDir);
}
@Test
public void testBasic() {
List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
// mock1Setting = factory.createNewSetting("mock11", "mock1", new ArrayList<Dependency>(), new InterpreterOption(false), new Properties());
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
// get interpreter
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
// try to get unavailable interpreter
assertNull(interpreterSettingManager.get("unknown"));
// restart interpreter
interpreterSettingManager.restart(mock1Setting.getId());
assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session"));
}
@Test
public void testRemoteRepl() throws Exception {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
intp1Properties.put("PROPERTY_1",
new InterpreterProperty("PROPERTY_1", "VALUE_1"));
intp1Properties.put("property_2", new InterpreterProperty("property_2", "value_2"));
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user", "sharedProcess");
factory.createInterpretersForNote(mock1Setting, "user", "sharedProcess", "session");
// get interpreter
assertNotNull("get Interpreter", interpreterGroup.get("session").get(0));
assertTrue(interpreterGroup.get("session").get(0) instanceof LazyOpenInterpreter);
LazyOpenInterpreter lazyInterpreter = (LazyOpenInterpreter)(interpreterGroup.get("session").get(0));
assertTrue(lazyInterpreter.getInnerInterpreter() instanceof RemoteInterpreter);
RemoteInterpreter remoteInterpreter = (RemoteInterpreter) lazyInterpreter.getInnerInterpreter();
assertEquals("VALUE_1", remoteInterpreter.getEnv().get("PROPERTY_1"));
assertEquals("value_2", remoteInterpreter.getProperty("property_2"));
}
/**
* 2 users' interpreters in scoped mode. Each user has one session. Restarting user1's interpreter
* won't affect user2's interpreter
* @throws Exception
*/
@Test
public void testRestartInterpreterInScopedMode() throws Exception {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
intp1Properties.put("PROPERTY_1",
new InterpreterProperty("PROPERTY_1", "VALUE_1"));
intp1Properties.put("property_2",
new InterpreterProperty("property_2", "value_2"));
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
mock1Setting.getOption().setPerUser("scoped");
mock1Setting.getOption().setPerNote("shared");
// set remote as false so that we won't create new remote interpreter process
mock1Setting.getOption().setRemote(false);
mock1Setting.getOption().setHost("localhost");
mock1Setting.getOption().setPort(2222);
InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("user1", "sharedProcess");
factory.createInterpretersForNote(mock1Setting, "user1", "sharedProcess", "user1");
factory.createInterpretersForNote(mock1Setting, "user2", "sharedProcess", "user2");
LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter)interpreterGroup.get("user1").get(0);
interpreter1.open();
LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup.get("user2").get(0);
interpreter2.open();
mock1Setting.closeAndRemoveInterpreterGroup("sharedProcess", "user1");
assertFalse(interpreter1.isOpen());
assertTrue(interpreter2.isOpen());
}
/**
* 2 users' interpreters in isolated mode. Each user has one interpreterGroup. Restarting user1's interpreter
* won't affect user2's interpreter
* @throws Exception
*/
@Test
public void testRestartInterpreterInIsolatedMode() throws Exception {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
Map<String, InterpreterProperty> intp1Properties = new HashMap<String, InterpreterProperty>();
intp1Properties.put("PROPERTY_1",
new InterpreterProperty("PROPERTY_1", "VALUE_1"));
intp1Properties.put("property_2",
new InterpreterProperty("property_2", "value_2"));
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(true), intp1Properties);
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
List<InterpreterSetting> all = interpreterSettingManager.get();
InterpreterSetting mock1Setting = null;
for (InterpreterSetting setting : all) {
if (setting.getName().equals("mock1")) {
mock1Setting = setting;
break;
}
}
mock1Setting.getOption().setPerUser("isolated");
mock1Setting.getOption().setPerNote("shared");
// set remote as false so that we won't create new remote interpreter process
mock1Setting.getOption().setRemote(false);
mock1Setting.getOption().setHost("localhost");
mock1Setting.getOption().setPort(2222);
InterpreterGroup interpreterGroup1 = mock1Setting.getInterpreterGroup("user1", "note1");
InterpreterGroup interpreterGroup2 = mock1Setting.getInterpreterGroup("user2", "note2");
factory.createInterpretersForNote(mock1Setting, "user1", "note1", "shared_session");
factory.createInterpretersForNote(mock1Setting, "user2", "note2", "shared_session");
LazyOpenInterpreter interpreter1 = (LazyOpenInterpreter)interpreterGroup1.get("shared_session").get(0);
interpreter1.open();
LazyOpenInterpreter interpreter2 = (LazyOpenInterpreter)interpreterGroup2.get("shared_session").get(0);
interpreter2.open();
mock1Setting.closeAndRemoveInterpreterGroup("note1", "user1");
assertFalse(interpreter1.isOpen());
assertTrue(interpreter2.isOpen());
}
@Test
public void testFactoryDefaultList() throws IOException, RepositoryException {
// get default settings
List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
assertTrue(interpreterSettingManager.get().size() >= all.size());
}
@Test
public void testExceptions() throws InterpreterException, IOException, RepositoryException {
List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList();
// add setting with null option & properties expected nullArgumentException.class
try {
interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
} catch(NullArgumentException e) {
assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
}
try {
interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null);
} catch (NullArgumentException e){
assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage());
}
}
@Test
public void testSaveLoad() throws IOException, RepositoryException {
// interpreter settings
int numInterpreters = interpreterSettingManager.get().size();
// check if file saved
assertTrue(new File(conf.getInterpreterSettingPath()).exists());
interpreterSettingManager.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new HashMap<String, InterpreterProperty>());
assertEquals(numInterpreters + 1, interpreterSettingManager.get().size());
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
/*
Current situation, if InterpreterSettinfRef doesn't have the key of InterpreterSetting, it would be ignored.
Thus even though interpreter.json have several interpreterSetting in that file, it would be ignored and would not be initialized from loadFromFile.
In this case, only "mock11" would be referenced from file under interpreter/mock, and "mock11" group would be initialized.
*/
// TODO(jl): Decide how to handle the know referenced interpreterSetting.
assertEquals(1, interpreterSettingManager.get().size());
}
@Test
public void testInterpreterSettingPropertyClass() throws IOException, RepositoryException {
// check if default interpreter reference's property type is map
Map<String, InterpreterSetting> interpreterSettingRefs = interpreterSettingManager.getAvailableInterpreterSettings();
InterpreterSetting intpSetting = interpreterSettingRefs.get("mock1");
Map<String, DefaultInterpreterProperty> intpProperties =
(Map<String, DefaultInterpreterProperty>) intpSetting.getProperties();
assertTrue(intpProperties instanceof Map);
// check if interpreter instance is saved as Properties in conf/interpreter.json file
Map<String, InterpreterProperty> properties = new HashMap<String, InterpreterProperty>();
properties.put("key1", new InterpreterProperty("key1", "value1", "type1"));
properties.put("key2", new InterpreterProperty("key2", "value2", "type2"));
interpreterSettingManager.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties);
String confFilePath = conf.getInterpreterSettingPath();
byte[] encoded = Files.readAllBytes(Paths.get(confFilePath));
String json = new String(encoded, "UTF-8");
InterpreterInfoSaving infoSaving = InterpreterInfoSaving.fromJson(json);
Map<String, InterpreterSetting> interpreterSettings = infoSaving.interpreterSettings;
for (String key : interpreterSettings.keySet()) {
InterpreterSetting setting = interpreterSettings.get(key);
if (setting.getName().equals("newMock")) {
assertEquals(setting.getProperties().toString(), properties.toString());
}
}
}
@Test
public void testInterpreterAliases() throws IOException, RepositoryException {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null);
interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>() {{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
interpreterSettingManager.add("group2", new ArrayList<InterpreterInfo>(){{
add(info2);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2", null);
final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
final InterpreterSetting setting2 = interpreterSettingManager.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
interpreterSettingManager.setInterpreters("user", "note", new ArrayList<String>() {{
add(setting1.getId());
add(setting2.getId());
}});
assertEquals("className1", factory.getInterpreter("user1", "note", "test-group1").getClassName());
assertEquals("className1", factory.getInterpreter("user1", "note", "group1").getClassName());
}
@Test
public void testMultiUser() throws IOException, RepositoryException {
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager);
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>(){{
add(info1);
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null);
InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED);
final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new HashMap<String, InterpreterProperty>());
interpreterSettingManager.setInterpreters("user1", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
interpreterSettingManager.setInterpreters("user2", "note", new ArrayList<String>() {{
add(setting1.getId());
}});
assertNotEquals(factory.getInterpreter("user1", "note", "test-group1"), factory.getInterpreter("user2", "note", "test-group1"));
}
@Test
public void testInvalidInterpreterSettingName() {
try {
interpreterSettingManager.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new HashMap<String, InterpreterProperty>());
fail("expect fail because of invalid InterpreterSetting Name");
} catch (IOException e) {
assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage());
}
}
@Test
public void getEditorSetting() throws IOException, RepositoryException, SchedulerException {
List<String> intpIds = new ArrayList<>();
for(InterpreterSetting intpSetting: interpreterSettingManager.get()) {
if (intpSetting.getName().startsWith("mock1")) {
intpIds.add(intpSetting.getId());
}
}
Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous"));
Interpreter interpreter = factory.getInterpreter("user1", note.getId(), "mock11");
// get editor setting from interpreter-setting.json
Map<String, Object> editor = interpreterSettingManager.getEditorSetting(interpreter, "user1", note.getId(), "mock11");
assertEquals("java", editor.get("language"));
// when interpreter is not loaded via interpreter-setting.json
// or editor setting doesn't exit
editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock1"),"user1", note.getId(), "mock1");
assertEquals(null, editor.get("language"));
// when interpreter is not bound to note
editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock11"),"user1", note.getId(), "mock2");
assertEquals("text", editor.get("language"));
}
@Test
public void registerCustomInterpreterRunner() throws IOException {
InterpreterSettingManager spyInterpreterSettingManager = spy(interpreterSettingManager);
doNothing().when(spyInterpreterSettingManager).saveToFile();
ArrayList<InterpreterInfo> interpreterInfos1 = new ArrayList<>();
interpreterInfos1.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
spyInterpreterSettingManager.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, DefaultInterpreterProperty>newHashMap(), "/normalGroup1", null);
spyInterpreterSettingManager.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap()));
InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
when(mockInterpreterRunner.getPath()).thenReturn("custom-linux-path.sh");
spyInterpreterSettingManager.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, DefaultInterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner);
spyInterpreterSettingManager.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new HashMap<String, InterpreterProperty>());
spyInterpreterSettingManager.setInterpreters("anonymous", "noteCustome", spyInterpreterSettingManager.getDefaultInterpreterSettingList());
factory.getInterpreter("anonymous", "noteCustome", "customGroup1");
verify(mockInterpreterRunner, times(1)).getPath();
}
@Test
public void interpreterRunnerTest() {
InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
String testInterpreterRunner = "relativePath.sh";
when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner); // This test only for Linux
Interpreter i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
String interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
assertNotEquals(interpreterRunner, testInterpreterRunner);
testInterpreterRunner = "/AbsolutePath.sh";
when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner);
i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
assertEquals(interpreterRunner, testInterpreterRunner);
}
}

327
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java
View file

@ -1,327 +0,0 @@
package org.apache.zeppelin.interpreter;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.junit.Test;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
public class InterpreterSettingTest {
@Test
public void sharedModeCloseandRemoveInterpreterGroupTest() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SHARED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
// This won't effect anything
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perUserScopedModeCloseAndRemoveInterpreterGroupTest() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(2, interpreterSetting.getInterpreterGroup("user2", "note1").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
// Check if non-existed key works or not
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perUserIsolatedModeCloseAndRemoveInterpreterGroupTest() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2);
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user2", "note1").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perNoteScopedModeCloseAndRemoveInterpreterGroupTest() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.SCOPED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note2");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note2"), interpreterList2);
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note2").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
// Check if non-existed key works or not
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note2", "user1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perNoteIsolatedModeCloseAndRemoveInterpreterGroupTest() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.ISOLATED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
Interpreter mockInterpreter2 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList2 = new ArrayList<>();
interpreterList2.add(mockInterpreter2);
interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note2");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note2"), interpreterList2);
assertEquals(2, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note2").size());
interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1");
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note2").size());
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
interpreterSetting.closeAndRemoveInterpreterGroup("note2", "user1");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
}
@Test
public void perNoteScopedModeRemoveInterpreterGroupWhenNoteIsRemoved() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.SCOPED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
// This method will be called when remove note
interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
// Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
assertEquals(0, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
@Test
public void perNoteIsolatedModeRemoveInterpreterGroupWhenNoteIsRemoved() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerNote(InterpreterOption.ISOLATED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
// This method will be called when remove note
interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
assertEquals(0, interpreterSetting.getAllInterpreterGroups().size());
// Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
assertEquals(0, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
@Test
public void perUserScopedModeNeverRemoveInterpreterGroupWhenNoteIsRemoved() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.SCOPED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
// This method will be called when remove note
interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
@Test
public void perUserIsolatedModeNeverRemoveInterpreterGroupWhenNoteIsRemoved() {
InterpreterOption interpreterOption = new InterpreterOption();
interpreterOption.setPerUser(InterpreterOption.ISOLATED);
InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList<InterpreterInfo>(), new Properties(), new ArrayList<Dependency>(), interpreterOption, "", null);
interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() {
@Override
public InterpreterGroup createInterpreterGroup(String interpreterGroupId,
InterpreterOption option) {
return new InterpreterGroup(interpreterGroupId);
}
});
Interpreter mockInterpreter1 = mock(RemoteInterpreter.class);
List<Interpreter> interpreterList1 = new ArrayList<>();
interpreterList1.add(mockInterpreter1);
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1");
interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1);
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size());
// This method will be called when remove note
interpreterSetting.closeAndRemoveInterpreterGroup("note1","");
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
// Be careful that getInterpreterGroup makes interpreterGroup if it doesn't exist
assertEquals(1, interpreterSetting.getInterpreterGroup("user1","note1").size());
}
}

131
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
View file

@ -1,131 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.*;
import java.util.HashMap;
import java.util.Properties;
import org.apache.thrift.TException;
import org.apache.thrift.transport.TTransportException;
import org.apache.zeppelin.interpreter.Constants;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.junit.Test;
public class RemoteInterpreterProcessTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private static final int DUMMY_PORT=3678;
@Test
public void testStartStop() {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
10 * 1000, null, null,"fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
assertEquals(2, rip.reference(intpGroup, "anonymous", false));
assertEquals(true, rip.isRunning());
assertEquals(1, rip.dereference());
assertEquals(true, rip.isRunning());
assertEquals(0, rip.dereference());
assertEquals(false, rip.isRunning());
}
@Test
public void testClientFactory() throws Exception {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class), 10 * 1000, "fakeName");
rip.reference(intpGroup, "anonymous", false);
assertEquals(0, rip.getNumActiveClient());
assertEquals(0, rip.getNumIdleClient());
Client client = rip.getClient();
assertEquals(1, rip.getNumActiveClient());
assertEquals(0, rip.getNumIdleClient());
rip.releaseClient(client);
assertEquals(0, rip.getNumActiveClient());
assertEquals(1, rip.getNumIdleClient());
rip.dereference();
}
@Test
public void testStartStopRemoteInterpreter() throws TException, InterruptedException {
RemoteInterpreterServer server = new RemoteInterpreterServer(3678);
server.start();
boolean running = false;
long startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < 10 * 1000) {
if (server.isRunning()) {
running = true;
break;
} else {
Thread.sleep(200);
}
}
Properties properties = new Properties();
properties.setProperty(Constants.ZEPPELIN_INTERPRETER_PORT, "3678");
properties.setProperty(Constants.ZEPPELIN_INTERPRETER_HOST, "localhost");
InterpreterGroup intpGroup = mock(InterpreterGroup.class);
when(intpGroup.getProperty()).thenReturn(properties);
when(intpGroup.containsKey(Constants.EXISTING_PROCESS)).thenReturn(true);
RemoteInterpreterProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT,
"nonexists",
"fakeRepo",
new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class)
, 10 * 1000,
"fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
assertEquals(true, rip.isRunning());
}
@Test
public void testPropagateError() throws TException, InterruptedException {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
"echo hello_world", "nonexists", "fakeRepo", new HashMap<String, String>(),
10 * 1000, null, null, "fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
try {
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
} catch (InterpreterException e) {
e.getMessage().contains("hello_world");
}
assertEquals(0, rip.referenceCount());
}
}

975
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
View file

@ -1,975 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.interpreter.remote;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.thrift.transport.TTransportException;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterEnv;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterB;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.Scheduler;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
public class RemoteInterpreterTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup;
private HashMap<String, String> env;
@Before
public void setUp() throws Exception {
intpGroup = new InterpreterGroup();
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
}
@After
public void tearDown() throws Exception {
intpGroup.close();
}
private RemoteInterpreter createMockInterpreterA(Properties p) {
return createMockInterpreterA(p, "note");
}
private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) {
return new RemoteInterpreter(
p,
noteId,
MockInterpreterA.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false);
}
private RemoteInterpreter createMockInterpreterB(Properties p) {
return createMockInterpreterB(p, "note");
}
private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) {
return new RemoteInterpreter(
p,
noteId,
MockInterpreterB.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false);
}
@Test
public void testRemoteInterperterCall() throws TTransportException, IOException {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreter intpB = createMockInterpreterB(p);
intpGroup.get("note").add(intpB);
intpB.setInterpreterGroup(intpGroup);
RemoteInterpreterProcess process = intpA.getInterpreterProcess();
process.equals(intpB.getInterpreterProcess());
assertFalse(process.isRunning());
assertEquals(0, process.getNumIdleClient());
assertEquals(0, process.referenceCount());
intpA.open(); // initializa all interpreters in the same group
assertTrue(process.isRunning());
assertEquals(1, process.getNumIdleClient());
assertEquals(1, process.referenceCount());
intpA.interpret("1",
new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
intpB.open();
assertEquals(1, process.referenceCount());
intpA.close();
assertEquals(0, process.referenceCount());
intpB.close();
assertEquals(0, process.referenceCount());
assertFalse(process.isRunning());
}
@Test
public void testExecuteIncorrectPrecode() throws TTransportException, IOException {
Properties p = new Properties();
p.put("zeppelin.MockInterpreterA.precode", "fail test");
intpGroup.put("note", new LinkedList<Interpreter>());
RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreterProcess process = intpA.getInterpreterProcess();
intpA.open();
InterpreterResult result = intpA.interpret("1",
new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
intpA.close();
assertEquals(Code.ERROR, result.code());
}
@Test
public void testExecuteCorrectPrecode() throws TTransportException, IOException {
Properties p = new Properties();
p.put("zeppelin.MockInterpreterA.precode", "2");
intpGroup.put("note", new LinkedList<Interpreter>());
RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreterProcess process = intpA.getInterpreterProcess();
intpA.open();
InterpreterResult result = intpA.interpret("1",
new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
intpA.close();
assertEquals(Code.SUCCESS, result.code());
assertEquals("1", result.message().get(0).getData());
}
@Test
public void testRemoteInterperterErrorStatus() throws TTransportException, IOException {
Properties p = new Properties();
RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
InterpreterResult ret = intpA.interpret("non numeric value",
new InterpreterContext(
"noteId",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
assertEquals(Code.ERROR, ret.code());
}
@Test
public void testRemoteSchedulerSharing() throws TTransportException, IOException {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
RemoteInterpreter intpA = new RemoteInterpreter(
p,
"note",
MockInterpreterA.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreter intpB = new RemoteInterpreter(
p,
"note",
MockInterpreterB.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false);
intpGroup.get("note").add(intpB);
intpB.setInterpreterGroup(intpGroup);
intpA.open();
intpB.open();
long start = System.currentTimeMillis();
InterpreterResult ret = intpA.interpret("500",
new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
assertEquals("500", ret.message().get(0).getData());
ret = intpB.interpret("500",
new InterpreterContext(
"note",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
assertEquals("1000", ret.message().get(0).getData());
long end = System.currentTimeMillis();
assertTrue(end - start >= 1000);
intpA.close();
intpB.close();
}
@Test
public void testRemoteSchedulerSharingSubmit() throws TTransportException, IOException, InterruptedException {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
final RemoteInterpreter intpB = createMockInterpreterB(p);
intpGroup.get("note").add(intpB);
intpB.setInterpreterGroup(intpGroup);
intpA.open();
intpB.open();
long start = System.currentTimeMillis();
Job jobA = new Job("jobA", null) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
return intpA.interpret("500",
new InterpreterContext(
"note",
"jobA",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
}
@Override
protected boolean jobAbort() {
return false;
}
};
intpA.getScheduler().submit(jobA);
Job jobB = new Job("jobB", null) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
return intpB.interpret("500",
new InterpreterContext(
"note",
"jobB",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
}
@Override
protected boolean jobAbort() {
return false;
}
};
intpB.getScheduler().submit(jobB);
// wait until both job finished
while (jobA.getStatus() != Status.FINISHED ||
jobB.getStatus() != Status.FINISHED) {
Thread.sleep(100);
}
long end = System.currentTimeMillis();
assertTrue(end - start >= 1000);
assertEquals("1000", ((InterpreterResult) jobB.getReturn()).message().get(0).getData());
intpA.close();
intpB.close();
}
@Test
public void testRunOrderPreserved() throws InterruptedException {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
int concurrency = 3;
final List<InterpreterResultMessage> results = new LinkedList<>();
Scheduler scheduler = intpA.getScheduler();
for (int i = 0; i < concurrency; i++) {
final String jobId = Integer.toString(i);
scheduler.submit(new Job(jobId, Integer.toString(i), null, 200) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
InterpreterResult ret = intpA.interpret(getJobName(), new InterpreterContext(
"note",
jobId,
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
synchronized (results) {
results.addAll(ret.message());
results.notify();
}
return null;
}
@Override
protected boolean jobAbort() {
return false;
}
});
}
// wait for job finished
synchronized (results) {
while (results.size() != concurrency) {
results.wait(300);
}
}
int i = 0;
for (InterpreterResultMessage result : results) {
assertEquals(Integer.toString(i++), result.getData());
}
assertEquals(concurrency, i);
intpA.close();
}
@Test
public void testRunParallel() throws InterruptedException {
Properties p = new Properties();
p.put("parallel", "true");
intpGroup.put("note", new LinkedList<Interpreter>());
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
int concurrency = 4;
final int timeToSleep = 1000;
final List<InterpreterResultMessage> results = new LinkedList<>();
long start = System.currentTimeMillis();
Scheduler scheduler = intpA.getScheduler();
for (int i = 0; i < concurrency; i++) {
final String jobId = Integer.toString(i);
scheduler.submit(new Job(jobId, Integer.toString(i), null, 300) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
String stmt = Integer.toString(timeToSleep);
InterpreterResult ret = intpA.interpret(stmt, new InterpreterContext(
"note",
jobId,
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
synchronized (results) {
results.addAll(ret.message());
results.notify();
}
return stmt;
}
@Override
protected boolean jobAbort() {
return false;
}
});
}
// wait for job finished
synchronized (results) {
while (results.size() != concurrency) {
results.wait(300);
}
}
long end = System.currentTimeMillis();
assertTrue(end - start < timeToSleep * concurrency);
intpA.close();
}
@Test
public void testInterpreterGroupResetBeforeProcessStarts() {
Properties p = new Properties();
RemoteInterpreter intpA = createMockInterpreterA(p);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
intpA.setInterpreterGroup(new InterpreterGroup(intpA.getInterpreterGroup().getId()));
RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
assertNotSame(processA.hashCode(), processB.hashCode());
}
@Test
public void testInterpreterGroupResetAfterProcessFinished() {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
RemoteInterpreter intpA = createMockInterpreterA(p);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
intpA.open();
processA.dereference(); // intpA.close();
intpA.setInterpreterGroup(new InterpreterGroup(intpA.getInterpreterGroup().getId()));
RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
assertNotSame(processA.hashCode(), processB.hashCode());
}
@Test
public void testInterpreterGroupResetDuringProcessRunning() throws InterruptedException {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
Job jobA = new Job("jobA", null) {
private Object r;
@Override
public Object getReturn() {
return r;
}
@Override
public void setResult(Object results) {
this.r = results;
}
@Override
public int progress() {
return 0;
}
@Override
public Map<String, Object> info() {
return null;
}
@Override
protected Object jobRun() throws Throwable {
return intpA.interpret("2000",
new InterpreterContext(
"note",
"jobA",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null));
}
@Override
protected boolean jobAbort() {
return false;
}
};
intpA.getScheduler().submit(jobA);
// wait for job started
while (intpA.getScheduler().getJobsRunning().size() == 0) {
Thread.sleep(100);
}
// restart interpreter
RemoteInterpreterProcess processA = intpA.getInterpreterProcess();
intpA.close();
InterpreterGroup newInterpreterGroup =
new InterpreterGroup(intpA.getInterpreterGroup().getId());
newInterpreterGroup.put("note", new LinkedList<Interpreter>());
intpA.setInterpreterGroup(newInterpreterGroup);
intpA.open();
RemoteInterpreterProcess processB = intpA.getInterpreterProcess();
assertNotSame(processA.hashCode(), processB.hashCode());
}
@Test
public void testRemoteInterpreterSharesTheSameSchedulerInstanceInTheSameGroup() {
Properties p = new Properties();
intpGroup.put("note", new LinkedList<Interpreter>());
RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
RemoteInterpreter intpB = createMockInterpreterB(p);
intpGroup.get("note").add(intpB);
intpB.setInterpreterGroup(intpGroup);
intpA.open();
intpB.open();
assertEquals(intpA.getScheduler(), intpB.getScheduler());
}
@Test
public void testMultiInterpreterSession() {
Properties p = new Properties();
intpGroup.put("sessionA", new LinkedList<Interpreter>());
intpGroup.put("sessionB", new LinkedList<Interpreter>());
RemoteInterpreter intpAsessionA = createMockInterpreterA(p, "sessionA");
intpGroup.get("sessionA").add(intpAsessionA);
intpAsessionA.setInterpreterGroup(intpGroup);
RemoteInterpreter intpBsessionA = createMockInterpreterB(p, "sessionA");
intpGroup.get("sessionA").add(intpBsessionA);
intpBsessionA.setInterpreterGroup(intpGroup);
intpAsessionA.open();
intpBsessionA.open();
assertEquals(intpAsessionA.getScheduler(), intpBsessionA.getScheduler());
RemoteInterpreter intpAsessionB = createMockInterpreterA(p, "sessionB");
intpGroup.get("sessionB").add(intpAsessionB);
intpAsessionB.setInterpreterGroup(intpGroup);
RemoteInterpreter intpBsessionB = createMockInterpreterB(p, "sessionB");
intpGroup.get("sessionB").add(intpBsessionB);
intpBsessionB.setInterpreterGroup(intpGroup);
intpAsessionB.open();
intpBsessionB.open();
assertEquals(intpAsessionB.getScheduler(), intpBsessionB.getScheduler());
assertNotEquals(intpAsessionA.getScheduler(), intpAsessionB.getScheduler());
}
@Test
public void should_push_local_angular_repo_to_remote() throws Exception {
//Given
final Client client = Mockito.mock(Client.class);
final RemoteInterpreter intr = new RemoteInterpreter(new Properties(), "noteId",
MockInterpreterA.class.getName(), "runner", "path", "localRepo", env, 10 * 1000, null,
null, "anonymous", false);
final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null);
registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId");
final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId");
interpreterGroup.setAngularObjectRegistry(registry);
intr.setInterpreterGroup(interpreterGroup);
final java.lang.reflect.Type registryType = new TypeToken<Map<String,
Map<String, AngularObject>>>() {}.getType();
final Gson gson = new Gson();
final String expected = gson.toJson(registry.getRegistry(), registryType);
//When
intr.pushAngularObjectRegistryToRemote(client);
//Then
Mockito.verify(client).angularRegistryPush(expected);
}
@Test
public void testEnvStringPattern() {
assertFalse(RemoteInterpreterUtils.isEnvString(null));
assertFalse(RemoteInterpreterUtils.isEnvString(""));
assertFalse(RemoteInterpreterUtils.isEnvString("abcDEF"));
assertFalse(RemoteInterpreterUtils.isEnvString("ABC-DEF"));
assertTrue(RemoteInterpreterUtils.isEnvString("ABCDEF"));
assertTrue(RemoteInterpreterUtils.isEnvString("ABC_DEF"));
assertTrue(RemoteInterpreterUtils.isEnvString("ABC_DEF123"));
}
@Test
public void testEnvronmentAndPropertySet() {
Properties p = new Properties();
p.setProperty("MY_ENV1", "env value 1");
p.setProperty("my.property.1", "property value 1");
RemoteInterpreter intp = new RemoteInterpreter(
p,
"note",
MockInterpreterEnv.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intp);
intp.setInterpreterGroup(intpGroup);
intp.open();
InterpreterContext context = new InterpreterContext(
"noteId",
"id",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
assertEquals("env value 1", intp.interpret("getEnv MY_ENV1", context).message().get(0).getData());
assertEquals(Code.ERROR, intp.interpret("getProperty MY_ENV1", context).code());
assertEquals(Code.ERROR, intp.interpret("getEnv my.property.1", context).code());
assertEquals("property value 1", intp.interpret("getProperty my.property.1", context).message().get(0).getData());
intp.close();
}
@Test
public void testSetProgress() throws InterruptedException {
// given MockInterpreterA set progress through InterpreterContext
Properties p = new Properties();
p.setProperty("progress", "50");
final RemoteInterpreter intpA = createMockInterpreterA(p);
intpGroup.put("note", new LinkedList<Interpreter>());
intpGroup.get("note").add(intpA);
intpA.setInterpreterGroup(intpGroup);
intpA.open();
final InterpreterContext context1 = new InterpreterContext(
"noteId",
"id1",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
InterpreterContext context2 = new InterpreterContext(
"noteId",
"id2",
null,
"title",
"text",
new AuthenticationInfo(),
new HashMap<String, Object>(),
new GUI(),
new AngularObjectRegistry(intpGroup.getId(), null),
new LocalResourcePool("pool1"),
new LinkedList<InterpreterContextRunner>(), null);
assertEquals(0, intpA.getProgress(context1));
assertEquals(0, intpA.getProgress(context2));
// when interpreter update progress through InterpreterContext
Thread t = new Thread() {
public void run() {
InterpreterResult ret = intpA.interpret("1000", context1);
}
};
t.start();
// then progress need to be updated in given context
while(intpA.getProgress(context1) == 0) Thread.yield();
assertEquals(50, intpA.getProgress(context1));
assertEquals(0, intpA.getProgress(context2));
t.join();
assertEquals(0, intpA.getProgress(context1));
assertEquals(0, intpA.getProgress(context2));
}
}

243
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
View file

@ -1,243 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.notebook;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter11;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
public class NoteInterpreterLoaderTest {
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterFactory factory;
private InterpreterSettingManager interpreterSettingManager;
private DependencyResolver depResolver;
@Before
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
new File(tmpDir, "conf").mkdirs();
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
conf = ZeppelinConfiguration.create();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, Maps.<String, Object>newHashMap()));
interpreterInfos.add(new InterpreterInfo(MockInterpreter11.class.getName(), "mock11", false, Maps.<String, Object>newHashMap()));
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, Maps.<String, Object>newHashMap()));
interpreterSettingManager.add("group1", interpreterInfos, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock", null);
interpreterSettingManager.add("group2", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(), Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock", null);
interpreterSettingManager.createNewSetting("group1", "group1", Lists.<Dependency>newArrayList(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
interpreterSettingManager.createNewSetting("group2", "group2", Lists.<Dependency>newArrayList(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
Interpreter.registeredInterpreters.clear();
}
@Test
public void testGetInterpreter() throws IOException {
interpreterSettingManager.setInterpreters("user", "note", interpreterSettingManager.getDefaultInterpreterSettingList());
// when there're no interpreter selection directive
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", null).getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", " ").getClassName());
// when group name is omitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "mock11").getClassName());
// when 'name' is ommitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2").getClassName());
// when nothing is ommitted
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("user", "note", "group1.mock1").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("user", "note", "group1.mock11").getClassName());
assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("user", "note", "group2.mock2").getClassName());
interpreterSettingManager.closeNote("user", "note");
}
@Test
public void testNoteSession() throws IOException {
interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
// interpreters are not created before accessing it
assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
factory.getInterpreter("user", "noteA", null).open();
factory.getInterpreter("user", "noteB", null).open();
assertTrue(
factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
// interpreters are created after accessing it
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("noteA"));
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("noteB"));
// invalid close
interpreterSettingManager.closeNote("user", "note");
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
// when
interpreterSettingManager.closeNote("user", "noteA");
interpreterSettingManager.closeNote("user", "noteB");
// interpreters are destroyed after close
assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "shared_process").get("noteA"));
assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "shared_process").get("noteB"));
}
@Test
public void testNotePerInterpreterProcess() throws IOException {
interpreterSettingManager.setInterpreters("user", "noteA", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("noteA").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
interpreterSettingManager.setInterpreters("user", "noteB", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("noteB").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
// interpreters are not created before accessing it
assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
factory.getInterpreter("user", "noteA", null).open();
factory.getInterpreter("user", "noteB", null).open();
// per note interpreter process
assertFalse(
factory.getInterpreter("user", "noteA", null).getInterpreterGroup().getId().equals(
factory.getInterpreter("user", "noteB", null).getInterpreterGroup().getId()));
// interpreters are created after accessing it
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
assertNotNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
// when
interpreterSettingManager.closeNote("user", "noteA");
interpreterSettingManager.closeNote("user", "noteB");
// interpreters are destroyed after close
assertNull(interpreterSettingManager.getInterpreterSettings("noteA").get(0).getInterpreterGroup("user", "noteA").get("shared_session"));
assertNull(interpreterSettingManager.getInterpreterSettings("noteB").get(0).getInterpreterGroup("user", "noteB").get("shared_session"));
}
@Test
public void testNoteInterpreterCloseForAll() throws IOException {
interpreterSettingManager.setInterpreters("user", "FitstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getOption().setPerNote(InterpreterOption.SCOPED);
interpreterSettingManager.setInterpreters("user", "yourFirstNote", interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getOption().setPerNote(InterpreterOption.ISOLATED);
// interpreters are not created before accessing it
assertNull(interpreterSettingManager.getInterpreterSettings("FitstNote").get(0).getInterpreterGroup("user", "FitstNote").get("FitstNote"));
assertNull(interpreterSettingManager.getInterpreterSettings("yourFirstNote").get(0).getInterpreterGroup("user", "yourFirstNote").get("yourFirstNote"));
Interpreter firstNoteIntp = factory.getInterpreter("user", "FitstNote", "group1.mock1");
Interpreter yourFirstNoteIntp = factory.getInterpreter("user", "yourFirstNote", "group1.mock1");
firstNoteIntp.open();
yourFirstNoteIntp.open();
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
interpreterSettingManager.closeNote("user", "FitstNote");
assertFalse(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
//reopen
firstNoteIntp.open();
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
// invalid check
interpreterSettingManager.closeNote("invalid", "Note");
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
// invalid contains value check
interpreterSettingManager.closeNote("u", "Note");
assertTrue(((LazyOpenInterpreter)firstNoteIntp).isOpen());
assertTrue(((LazyOpenInterpreter)yourFirstNoteIntp).isOpen());
}
private void delete(File file){
if(file.isFile()) file.delete();
else if(file.isDirectory()){
File [] files = file.listFiles();
if(files!=null && files.length>0){
for(File f : files){
delete(f);
}
}
file.delete();
}
}
}

179
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
View file

@ -17,31 +17,27 @@
package org.apache.zeppelin.notebook;
import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
import com.google.common.collect.Maps;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.apache.zeppelin.resource.LocalResourcePool;
import org.apache.zeppelin.resource.ResourcePoolUtils;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.SchedulerFactory;
@ -56,18 +52,35 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositoryException;
public class NotebookTest implements JobListenerFactory{
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
public class NotebookTest extends AbstractInterpreterTest implements JobListenerFactory {
private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class);
private File tmpDir;
private ZeppelinConfiguration conf;
private SchedulerFactory schedulerFactory;
private File notebookDir;
private Notebook notebook;
private NotebookRepo notebookRepo;
private InterpreterFactory factory;
private InterpreterSettingManager interpreterSettingManager;
private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
private Credentials credentials;
private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS;
@ -75,57 +88,30 @@ public class NotebookTest implements JobListenerFactory{
@Before
public void setUp() throws Exception {
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true");
System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
super.setUp();
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
new File(tmpDir, "conf").mkdirs();
notebookDir = new File(tmpDir + "/notebook");
notebookDir.mkdirs();
System.setProperty(ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.toString() + "/conf");
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
conf = ZeppelinConfiguration.create();
this.schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(false));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>();
interpreterInfos2.add(new InterpreterInfo(MockInterpreter2.class.getName(), "mock2", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock2", interpreterInfos2, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock2", null);
interpreterSettingManager.createNewSetting("mock2", "mock2", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
schedulerFactory = SchedulerFactory.singleton();
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath());
notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search,
notebookAuthorization, credentials);
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true");
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
super.tearDown();
}
@Test
public void testSelectingReplImplementation() throws IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// run with default repl
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -259,7 +245,7 @@ public class NotebookTest implements JobListenerFactory{
Notebook notebook2 = new Notebook(
conf, notebookRepo, schedulerFactory,
new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager),
new InterpreterFactory(interpreterSettingManager),
interpreterSettingManager, null, null, null, null);
assertEquals(1, notebook2.getAllNotes().size());
@ -316,7 +302,7 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testRunAll() throws IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// p1
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -355,7 +341,7 @@ public class NotebookTest implements JobListenerFactory{
public void testSchedule() throws InterruptedException, IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
@ -428,8 +414,8 @@ public class NotebookTest implements JobListenerFactory{
public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map config = new HashMap<>();
p.setConfig(config);
@ -449,11 +435,11 @@ public class NotebookTest implements JobListenerFactory{
MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter)
((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter())
((LazyOpenInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock1")).getInnerInterpreter())
.getInnerInterpreter()));
MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter)
((LazyOpenInterpreter) factory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter())
((LazyOpenInterpreter) interpreterFactory.getInterpreter(anonymous.getUser(), note.getId(), "mock2")).getInnerInterpreter())
.getInnerInterpreter()));
// wait until interpreters are started
@ -467,9 +453,9 @@ public class NotebookTest implements JobListenerFactory{
}
// remove cron scheduler.
config.put("cron", null);
note.setConfig(config);
notebook.refreshCron(note.getId());
// config.put("cron", null);
// note.setConfig(config);
// notebook.refreshCron(note.getId());
// make sure all paragraph has been executed
assertNotNull(p.getDateFinished());
@ -481,7 +467,7 @@ public class NotebookTest implements JobListenerFactory{
public void testExportAndImportNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
String simpleText = "hello world";
@ -520,7 +506,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNote() throws IOException, CloneNotSupportedException,
InterruptedException, InterpreterException, SchedulerException, RepositoryException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@ -554,7 +540,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithNoName() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Note cloneNote = notebook.cloneNote(note.getId(), null, anonymous);
assertEquals(cloneNote.getName(), "Note " + cloneNote.getId());
@ -566,7 +552,7 @@ public class NotebookTest implements JobListenerFactory{
public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException,
InterruptedException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
final Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p.setText("hello world");
@ -591,28 +577,28 @@ public class NotebookTest implements JobListenerFactory{
@Test
public void testResourceRemovealOnParagraphNoteRemove() throws IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) {
intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
}
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello");
Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p2.setText("%mock2 world");
for (InterpreterGroup intpGroup : interpreterSettingManager.getAllInterpreterGroup()) {
intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId()));
}
note.runAll();
while (p1.isTerminated() == false || p1.getResult() == null) Thread.yield();
while (p2.isTerminated() == false || p2.getResult() == null) Thread.yield();
assertEquals(2, ResourcePoolUtils.getAllResources().size());
assertEquals(2, interpreterSettingManager.getAllResources().size());
// remove a paragraph
note.removeParagraph(anonymous.getUser(), p1.getId());
assertEquals(1, ResourcePoolUtils.getAllResources().size());
assertEquals(1, interpreterSettingManager.getAllResources().size());
// remove note
notebook.removeNote(note.getId(), anonymous);
assertEquals(0, ResourcePoolUtils.getAllResources().size());
assertEquals(0, interpreterSettingManager.getAllResources().size());
}
@Test
@ -620,10 +606,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -653,10 +639,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -687,10 +673,10 @@ public class NotebookTest implements JobListenerFactory{
IOException {
// create a note and a paragraph
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
AngularObjectRegistry registry = interpreterSettingManager
.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getInterpreterSettings(note.getId()).get(0).getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// add local scope object
@ -700,14 +686,13 @@ public class NotebookTest implements JobListenerFactory{
// restart interpreter
interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId());
registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
registry = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0)
.getOrCreateInterpreterGroup(anonymous.getUser(), "sharedProcess")
.getAngularObjectRegistry();
// local and global scope object should be removed
// But InterpreterGroup does not implement angularObjectRegistry per session (scoped, isolated)
// So for now, does not have good way to remove all objects in particular session on restart.
assertNotNull(registry.get("o1", note.getId(), null));
assertNotNull(registry.get("o2", null, null));
// New InterpreterGroup will be created and its AngularObjectRegistry will be created
assertNull(registry.get("o1", note.getId(), null));
assertNull(registry.get("o2", null, null));
notebook.removeNote(note.getId(), anonymous);
}
@ -802,7 +787,7 @@ public class NotebookTest implements JobListenerFactory{
public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException,
IOException {
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters(anonymous.getUser(), note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding(anonymous.getUser(), note.getId(), interpreterSettingManager.getInterpreterSettingIds());
// create three paragraphs
Paragraph p1 = note.addNewParagraph(anonymous);
@ -826,11 +811,11 @@ public class NotebookTest implements JobListenerFactory{
interpreterSettingManager.restart(interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getId());
// make sure three differnt status aborted well.
assertEquals(Status.FINISHED, p1.getStatus());
assertEquals(Status.ABORT, p2.getStatus());
assertEquals(Status.ABORT, p3.getStatus());
notebook.removeNote(note.getId(), anonymous);
// assertEquals(Status.FINISHED, p1.getStatus());
// assertEquals(Status.ABORT, p2.getStatus());
// assertEquals(Status.ABORT, p3.getStatus());
//
// notebook.removeNote(note.getId(), anonymous);
}
@Test

9
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
View file

@ -21,6 +21,7 @@ package org.apache.zeppelin.notebook;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
@ -39,7 +40,7 @@ import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectBuilder;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.Interpreter.FormType;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterGroup;
@ -47,10 +48,7 @@ import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSetting.Status;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.resource.ResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
@ -58,6 +56,7 @@ import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
import org.mockito.Mockito;
public class ParagraphTest {
@ -186,7 +185,7 @@ public class ParagraphTest {
when(mockInterpreterOption.permissionIsSet()).thenReturn(false);
when(mockInterpreterSetting.getStatus()).thenReturn(Status.READY);
when(mockInterpreterSetting.getId()).thenReturn("mock_id_1");
when(mockInterpreterSetting.getInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
when(mockInterpreterSetting.getOrCreateInterpreterGroup(anyString(), anyString())).thenReturn(mockInterpreterGroup);
spyInterpreterSettingList.add(mockInterpreterSetting);
when(mockNote.getId()).thenReturn("any_id");
when(mockInterpreterSettingManager.getInterpreterSettings(anyString())).thenReturn(spyInterpreterSettingList);

74
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
View file

@ -33,10 +33,13 @@ import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.notebook.*;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
@ -69,7 +72,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
private Credentials credentials;
private AuthenticationInfo anonymous;
private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
@Before
public void setUp() throws Exception {
String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis();
@ -91,12 +94,13 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
LOG.info("secondary note dir : " + secNotePath);
conf = ZeppelinConfiguration.create();
this.schedulerFactory = new SchedulerFactory();
this.schedulerFactory = SchedulerFactory.singleton();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
interpreterSettingManager = new InterpreterSettingManager(conf,
mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
factory = new InterpreterFactory(interpreterSettingManager);
search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
@ -110,19 +114,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
public void tearDown() throws Exception {
delete(mainZepDir);
}
@Test
public void testRepoCount() throws IOException {
assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount());
}
@Test
public void testSyncOnCreate() throws IOException {
/* check that both storage systems are empty */
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
/* create note */
Note note = notebookSync.createNote(anonymous);
@ -130,7 +134,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous);
}
@ -140,26 +144,26 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertTrue(notebookRepoSync.getRepoCount() > 1);
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
Note note = notebookSync.createNote(anonymous);
/* check that created in both storage systems */
assertEquals(1, notebookRepoSync.list(0, anonymous).size());
assertEquals(1, notebookRepoSync.list(1, anonymous).size());
assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
/* remove Note */
notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous);
/* check that deleted in both storages */
assertEquals(0, notebookRepoSync.list(0, anonymous).size());
assertEquals(0, notebookRepoSync.list(1, anonymous).size());
}
@Test
public void testSyncUpdateMain() throws IOException {
/* create note */
Note note = notebookSync.createNote(anonymous);
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
@ -167,19 +171,19 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
config.put("enabled", true);
p1.setConfig(config);
p1.setText("hello world");
/* new paragraph exists in note instance */
assertEquals(1, note.getParagraphs().size());
/* new paragraph not yet saved into storages */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
/* save to storage under index 0 (first storage) */
/* save to storage under index 0 (first storage) */
notebookRepoSync.save(0, note, anonymous);
/* check paragraph saved to first storage */
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
@ -284,45 +288,45 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
// one git versioned storage initialized
assertThat(vRepoSync.getRepoCount()).isEqualTo(1);
assertThat(vRepoSync.getRepo(0)).isInstanceOf(GitNotebookRepo.class);
GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0);
// no notes
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(0);
// create note
Note note = vNotebookSync.createNote(anonymous);
assertThat(vRepoSync.list(anonymous).size()).isEqualTo(1);
String noteId = vRepoSync.list(anonymous).get(0).getId();
// first checkpoint
vRepoSync.checkpoint(noteId, "checkpoint message", anonymous);
int vCount = gitRepo.revisionHistory(noteId, anonymous).size();
assertThat(vCount).isEqualTo(1);
Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p.getConfig();
config.put("enabled", true);
p.setConfig(config);
p.setText("%md checkpoint test");
// save and checkpoint again
vRepoSync.save(note, anonymous);
vRepoSync.checkpoint(noteId, "checkpoint message 2", anonymous);
assertThat(gitRepo.revisionHistory(noteId, anonymous).size()).isEqualTo(vCount + 1);
notebookRepoSync.remove(note.getId(), anonymous);
}
@Test
public void testSyncWithAcl() throws IOException {
/* scenario 1 - note exists with acl on main storage */
AuthenticationInfo user1 = new AuthenticationInfo("user1");
Note note = notebookSync.createNote(user1);
assertEquals(0, note.getParagraphs().size());
// saved on both storages
assertEquals(1, notebookRepoSync.list(0, null).size());
assertEquals(1, notebookRepoSync.list(1, null).size());
/* check that user1 is the only owner */
NotebookAuthorization authInfo = NotebookAuthorization.getInstance();
Set<String> entity = new HashSet<String>();
@ -331,23 +335,23 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, authInfo.getOwners(note.getId()).size());
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
/* update note and save on secondary storage */
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
p1.setText("hello world");
assertEquals(1, note.getParagraphs().size());
notebookRepoSync.save(1, note, null);
/* check paragraph isn't saved into first storage */
assertEquals(0, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
/* check paragraph is saved into second storage */
assertEquals(1, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
/* now sync by user1 */
notebookRepoSync.sync(user1);
/* check that note updated and acl are same on main storage*/
assertEquals(1, notebookRepoSync.get(0,
notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
@ -355,7 +359,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(1, authInfo.getOwners(note.getId()).size());
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
/* scenario 2 - note doesn't exist on main storage */
/* remove from main storage */
notebookRepoSync.remove(0, note.getId(), user1);
@ -365,7 +369,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
assertEquals(0, authInfo.getOwners(note.getId()).size());
assertEquals(0, authInfo.getReaders(note.getId()).size());
assertEquals(0, authInfo.getWriters(note.getId()).size());
/* now sync - should bring note from secondary storage with added acl */
notebookRepoSync.sync(user1);
assertEquals(1, notebookRepoSync.list(0, null).size());
@ -423,5 +427,5 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
}
};
}
}

55
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
View file

@ -22,18 +22,15 @@ import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.AbstractInterpreterTest;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterInfo;
import org.apache.zeppelin.interpreter.InterpreterOption;
@ -41,6 +38,7 @@ import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterProperty;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
@ -58,59 +56,34 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableMap;
public class VFSNotebookRepoTest implements JobListenerFactory {
public class VFSNotebookRepoTest extends AbstractInterpreterTest implements JobListenerFactory {
private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepoTest.class);
private ZeppelinConfiguration conf;
private SchedulerFactory schedulerFactory;
private Notebook notebook;
private NotebookRepo notebookRepo;
private InterpreterSettingManager interpreterSettingManager;
private InterpreterFactory factory;
private DependencyResolver depResolver;
private NotebookAuthorization notebookAuthorization;
private File mainZepDir;
private File mainNotebookDir;
@Before
public void setUp() throws Exception {
String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis();
mainZepDir = new File(zpath);
mainZepDir.mkdirs();
new File(mainZepDir, "conf").mkdirs();
String mainNotePath = zpath + "/notebook";
mainNotebookDir = new File(mainNotePath);
mainNotebookDir.mkdirs();
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2");
System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo");
conf = ZeppelinConfiguration.create();
this.schedulerFactory = new SchedulerFactory();
this.schedulerFactory = new SchedulerFactory();
depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true));
factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager);
ArrayList<InterpreterInfo> interpreterInfos = new ArrayList<>();
interpreterInfos.add(new InterpreterInfo(MockInterpreter1.class.getName(), "mock1", true, new HashMap<String, Object>()));
interpreterSettingManager.add("mock1", interpreterInfos, new ArrayList<Dependency>(), new InterpreterOption(),
Maps.<String, DefaultInterpreterProperty>newHashMap(), "mock1", null);
interpreterSettingManager.createNewSetting("mock1", "mock1", new ArrayList<Dependency>(), new InterpreterOption(), new HashMap<String, InterpreterProperty>());
super.setUp();
this.schedulerFactory = SchedulerFactory.singleton();
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
notebookAuthorization = NotebookAuthorization.init(conf);
notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, this, search,
notebook = new Notebook(conf, notebookRepo, schedulerFactory, interpreterFactory, interpreterSettingManager, this, search,
notebookAuthorization, null);
}
@After
public void tearDown() throws Exception {
if (!FileUtils.deleteQuietly(mainZepDir)) {
LOG.error("Failed to delete {} ", mainZepDir.getName());
if (!FileUtils.deleteQuietly(testRootDir)) {
LOG.error("Failed to delete {} ", testRootDir.getName());
}
}
@ -120,7 +93,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
int numNotes = notebookRepo.list(null).size();
// when create invalid json file
File testNoteDir = new File(mainNotebookDir, "test");
File testNoteDir = new File(notebookDir, "test");
testNoteDir.mkdir();
FileUtils.writeStringToFile(new File(testNoteDir, "note.json"), "");
@ -132,7 +105,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
public void testSaveNotebook() throws IOException, InterruptedException {
AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
Note note = notebook.createNote(anonymous);
interpreterSettingManager.setInterpreters("user", note.getId(), interpreterSettingManager.getDefaultInterpreterSettingList());
interpreterSettingManager.setInterpreterBinding("user", note.getId(), interpreterSettingManager.getInterpreterSettingIds());
Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
Map<String, Object> config = p1.getConfig();

96
zeppelin-zengine/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java
View file

@ -17,35 +17,34 @@
package org.apache.zeppelin.resource;
import com.google.gson.Gson;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller;
import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Properties;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
/**
* Unittest for DistributedResourcePool
*/
public class DistributedResourcePoolTest {
private static final String INTERPRETER_SCRIPT =
System.getProperty("os.name").startsWith("Windows") ?
"../bin/interpreter.cmd" :
"../bin/interpreter.sh";
private InterpreterGroup intpGroup1;
private InterpreterGroup intpGroup2;
private HashMap<String, String> env;
public class DistributedResourcePoolTest extends AbstractInterpreterTest {
private RemoteInterpreter intp1;
private RemoteInterpreter intp2;
private InterpreterContext context;
@ -55,50 +54,10 @@ public class DistributedResourcePoolTest {
@Before
public void setUp() throws Exception {
env = new HashMap<>();
env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath());
Properties p = new Properties();
intp1 = new RemoteInterpreter(
p,
"note",
MockInterpreterResourcePool.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false
);
intpGroup1 = new InterpreterGroup("intpGroup1");
intpGroup1.put("note", new LinkedList<Interpreter>());
intpGroup1.get("note").add(intp1);
intp1.setInterpreterGroup(intpGroup1);
intp2 = new RemoteInterpreter(
p,
"note",
MockInterpreterResourcePool.class.getName(),
new File(INTERPRETER_SCRIPT).getAbsolutePath(),
"fake",
"fakeRepo",
env,
10 * 1000,
null,
null,
"anonymous",
false
);
intpGroup2 = new InterpreterGroup("intpGroup2");
intpGroup2.put("note", new LinkedList<Interpreter>());
intpGroup2.get("note").add(intp2);
intp2.setInterpreterGroup(intpGroup2);
super.setUp();
InterpreterSetting interpreterSetting = interpreterSettingManager.getByName("mock_resource_pool");
intp1 = (RemoteInterpreter) interpreterSetting.getInterpreter("user1", "note1", "mock_resource_pool");
intp2 = (RemoteInterpreter) interpreterSetting.getInterpreter("user2", "note1", "mock_resource_pool");
context = new InterpreterContext(
"note",
@ -117,26 +76,13 @@ public class DistributedResourcePoolTest {
intp1.open();
intp2.open();
eventPoller1 = new RemoteInterpreterEventPoller(null, null);
eventPoller1.setInterpreterGroup(intpGroup1);
eventPoller1.setInterpreterProcess(intpGroup1.getRemoteInterpreterProcess());
eventPoller2 = new RemoteInterpreterEventPoller(null, null);
eventPoller2.setInterpreterGroup(intpGroup2);
eventPoller2.setInterpreterProcess(intpGroup2.getRemoteInterpreterProcess());
eventPoller1.start();
eventPoller2.start();
eventPoller1 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller();
eventPoller2 = intp1.getInterpreterGroup().getRemoteInterpreterProcess().getRemoteInterpreterEventPoller();
}
@After
public void tearDown() throws Exception {
eventPoller1.shutdown();
intp1.close();
intpGroup1.close();
eventPoller2.shutdown();
intp2.close();
intpGroup2.close();
interpreterSettingManager.close();
}
@Test
@ -235,13 +181,13 @@ public class DistributedResourcePoolTest {
// then get all resources.
assertEquals(4, ResourcePoolUtils.getAllResources().size());
assertEquals(4, interpreterSettingManager.getAllResources().size());
// when remove all resources from note1
ResourcePoolUtils.removeResourcesBelongsToNote("note1");
interpreterSettingManager.removeResourcesBelongsToNote("note1");
// then resources should be removed.
assertEquals(2, ResourcePoolUtils.getAllResources().size());
assertEquals(2, interpreterSettingManager.getAllResources().size());
assertEquals("", gson.fromJson(
intp1.interpret("get note1:paragraph1:key1", context).message().get(0).getData(),
String.class));
@ -251,10 +197,10 @@ public class DistributedResourcePoolTest {
// when remove all resources from note2:paragraph1
ResourcePoolUtils.removeResourcesBelongsToParagraph("note2", "paragraph1");
interpreterSettingManager.removeResourcesBelongsToParagraph("note2", "paragraph1");
// then 1
assertEquals(1, ResourcePoolUtils.getAllResources().size());
assertEquals(1, interpreterSettingManager.getAllResources().size());
assertEquals("value2", gson.fromJson(
intp1.interpret("get note2:paragraph2:key2", context).message().get(0).getData(),
String.class));

1
zeppelin-zengine/src/test/resources/conf/interpreter.json Normal file
View file

@ -0,0 +1 @@
{}

12
zeppelin-zengine/src/test/resources/interpreter/mock/interpreter-setting.json
View file

@ -1,12 +0,0 @@
[
{
"group": "mock11",
"name": "mock11",
"className": "org.apache.zeppelin.interpreter.mock.MockInterpreter11",
"properties": {
},
"editor": {
"language": "java"
}
}
]

19
zeppelin-zengine/src/test/resources/interpreter/mock1/interpreter-setting.json Normal file
View file

@ -0,0 +1,19 @@
[
{
"group": "mock1",
"name": "mock1",
"className": "org.apache.zeppelin.interpreter.mock.MockInterpreter1",
"properties": {
},
"option": {
"remote": false,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
]

19
zeppelin-zengine/src/test/resources/interpreter/mock2/interpreter-setting.json Normal file
View file

@ -0,0 +1,19 @@
[
{
"group": "mock2",
"name": "mock2",
"className": "org.apache.zeppelin.interpreter.mock.MockInterpreter2",
"properties": {
},
"option": {
"remote": false,
"port": -1,
"perNote": "shared",
"perUser": "isolated",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
]

19
zeppelin-zengine/src/test/resources/interpreter/mock_resource_pool/interpreter-setting.json Normal file
View file

@ -0,0 +1,19 @@
[
{
"group": "mock_resource_pool",
"name": "mock_resource_pool",
"className": "org.apache.zeppelin.interpreter.remote.mock.MockInterpreterResourcePool",
"properties": {
},
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "shared",
"isExistingProcess": false,
"setPermission": false,
"users": [],
"isUserImpersonate": false
}
}
]

3
zeppelin-zengine/src/test/resources/log4j.properties
View file

@ -35,7 +35,6 @@ log4j.logger.org.apache.hadoop.mapred=WARN
log4j.logger.org.apache.hadoop.hive.ql=WARN
log4j.logger.org.apache.hadoop.hive.metastore=WARN
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
log4j.logger.org.apache.zeppelin.scheduler=WARN
log4j.logger.org.quartz=WARN
log4j.logger.DataNucleus=WARN
@ -45,4 +44,6 @@ log4j.logger.DataNucleus.Datastore=ERROR
# Log all JDBC parameters
log4j.logger.org.hibernate.type=ALL
log4j.logger.org.apache.zeppelin.interpreter=DEBUG
log4j.logger.org.apache.zeppelin.scheduler=DEBUG