mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Show description when create new interpreter
This commit is contained in:
Mina Lee
parent
2f79852d34
commit
1a2a41dc44
5 changed files with 85 additions and 89 deletions
|
|
@ -23,6 +23,7 @@ import java.lang.ref.WeakReference;
|
|||
import java.net.InetAddress;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
|
|
@ -140,47 +141,41 @@ public abstract class AbstractTestRestApi {
|
|||
LOG.info("Test Zeppelin stared.");
|
||||
|
||||
|
||||
// assume first one is spark
|
||||
InterpreterSetting sparkIntpSetting = null;
|
||||
for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
|
||||
if (intpSetting.getName().equals("spark")) {
|
||||
sparkIntpSetting = intpSetting;
|
||||
}
|
||||
}
|
||||
|
||||
Properties sparkProperties = (Properties) sparkIntpSetting.getProperties();
|
||||
// ci environment runs spark cluster for testing
|
||||
// so configure zeppelin use spark cluster
|
||||
if ("true".equals(System.getenv("CI"))) {
|
||||
// assume first one is spark
|
||||
InterpreterSetting sparkIntpSetting = null;
|
||||
for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
|
||||
if (intpSetting.getName().equals("spark")) {
|
||||
sparkIntpSetting = intpSetting;
|
||||
}
|
||||
}
|
||||
|
||||
// set spark master and other properties
|
||||
sparkIntpSetting.getProperties().setProperty("master", "local[2]");
|
||||
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
|
||||
sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false");
|
||||
sparkProperties.setProperty("master", "local[2]");
|
||||
sparkProperties.setProperty("spark.cores.max", "2");
|
||||
sparkProperties.setProperty("zeppelin.spark.useHiveContext", "false");
|
||||
// set spark home for pyspark
|
||||
sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome());
|
||||
sparkProperties.setProperty("spark.home", getSparkHome());
|
||||
|
||||
sparkIntpSetting.setProperties(sparkProperties);
|
||||
pySpark = true;
|
||||
sparkR = true;
|
||||
ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId());
|
||||
} else {
|
||||
// assume first one is spark
|
||||
InterpreterSetting sparkIntpSetting = null;
|
||||
for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) {
|
||||
if (intpSetting.getName().equals("spark")) {
|
||||
sparkIntpSetting = intpSetting;
|
||||
}
|
||||
}
|
||||
|
||||
String sparkHome = getSparkHome();
|
||||
if (sparkHome != null) {
|
||||
if (System.getenv("SPARK_MASTER") != null) {
|
||||
sparkIntpSetting.getProperties().setProperty("master", System.getenv("SPARK_MASTER"));
|
||||
sparkProperties.setProperty("master", System.getenv("SPARK_MASTER"));
|
||||
} else {
|
||||
sparkIntpSetting.getProperties()
|
||||
.setProperty("master", "local[2]");
|
||||
sparkProperties.setProperty("master", "local[2]");
|
||||
}
|
||||
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
|
||||
sparkProperties.setProperty("spark.cores.max", "2");
|
||||
// set spark home for pyspark
|
||||
sparkIntpSetting.getProperties().setProperty("spark.home", sparkHome);
|
||||
sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false");
|
||||
sparkProperties.setProperty("spark.home", sparkHome);
|
||||
sparkProperties.setProperty("zeppelin.spark.useHiveContext", "false");
|
||||
pySpark = true;
|
||||
sparkR = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -399,12 +399,11 @@
|
|||
var intpInfo = el[i];
|
||||
for (var key in intpInfo) {
|
||||
properties[key] = {
|
||||
value: intpInfo[key],
|
||||
value: intpInfo[key].defaultValue,
|
||||
description: intpInfo[key].description
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
$scope.newInterpreterSetting.properties = properties;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -237,8 +237,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
interpreterInfo =
|
||||
new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(),
|
||||
r.getEditor());
|
||||
add(r.getGroup(), interpreterInfo, convertInterpreterProperties(r.getProperties()),
|
||||
r.getPath());
|
||||
add(r.getGroup(), interpreterInfo, r.getProperties(), r.getPath());
|
||||
}
|
||||
|
||||
for (String settingId : interpreterSettingsRef.keySet()) {
|
||||
|
|
@ -286,7 +285,8 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
|
||||
private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) {
|
||||
InterpreterSetting setting =
|
||||
new InterpreterSetting(o.getName(), o.getName(), o.getInterpreterInfos(), o.getProperties(),
|
||||
new InterpreterSetting(o.getName(), o.getName(), o.getInterpreterInfos(),
|
||||
convertInterpreterProperties((Map <String, InterpreterProperty>)o.getProperties()),
|
||||
o.getDependencies(), o.getOption(), o.getPath());
|
||||
setting.setInterpreterGroupFactory(this);
|
||||
return setting;
|
||||
|
|
@ -349,16 +349,9 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
InterpreterInfo interpreterInfo =
|
||||
new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(),
|
||||
registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor());
|
||||
Properties properties = new Properties();
|
||||
Map<String, InterpreterProperty> p = registeredInterpreter.getProperties();
|
||||
|
||||
if (null != p) {
|
||||
for (String key : p.keySet()) {
|
||||
properties.setProperty(key, p.get(key).getValue());
|
||||
}
|
||||
}
|
||||
|
||||
add(registeredInterpreter.getGroup(), interpreterInfo, properties, absolutePath);
|
||||
add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(),
|
||||
absolutePath);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -615,11 +608,11 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
}
|
||||
|
||||
private InterpreterSetting add(String group, InterpreterInfo interpreterInfo,
|
||||
Properties properties, String path)
|
||||
Map<String, InterpreterProperty> interpreterProperties, String path)
|
||||
throws InterpreterException, IOException, RepositoryException {
|
||||
ArrayList<InterpreterInfo> infos = new ArrayList<>();
|
||||
infos.add(interpreterInfo);
|
||||
return add(group, infos, new ArrayList<Dependency>(), defaultOption, properties, path);
|
||||
return add(group, infos, new ArrayList<Dependency>(), defaultOption, interpreterProperties, path);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -627,12 +620,13 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
* @return
|
||||
*/
|
||||
public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos,
|
||||
List<Dependency> dependencies, InterpreterOption option, Properties properties, String path) {
|
||||
List<Dependency> dependencies, InterpreterOption option,
|
||||
Map<String, InterpreterProperty> interpreterProperties, String path) {
|
||||
Preconditions.checkNotNull(group, "name should not be null");
|
||||
Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null");
|
||||
Preconditions.checkNotNull(dependencies, "dependencies should not be null");
|
||||
Preconditions.checkNotNull(option, "option should not be null");
|
||||
Preconditions.checkNotNull(properties, "properties should not be null");
|
||||
Preconditions.checkNotNull(interpreterProperties, "properties should not be null");
|
||||
|
||||
InterpreterSetting interpreterSetting;
|
||||
|
||||
|
|
@ -663,16 +657,17 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
}
|
||||
|
||||
// Append properties
|
||||
Properties interpreterProperties = interpreterSetting.getProperties();
|
||||
for (String key : properties.stringPropertyNames()) {
|
||||
if (!interpreterProperties.containsKey(key)) {
|
||||
interpreterProperties.setProperty(key, properties.getProperty(key));
|
||||
Map<String, InterpreterProperty> properties =
|
||||
(Map<String, InterpreterProperty>) interpreterSetting.getProperties();
|
||||
for (String key : interpreterProperties.keySet()) {
|
||||
if (!properties.containsKey(key)) {
|
||||
properties.put(key, interpreterProperties.get(key));
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
interpreterSetting =
|
||||
new InterpreterSetting(group, null, interpreterInfos, properties, dependencies, option,
|
||||
new InterpreterSetting(group, null, interpreterInfos, interpreterProperties, dependencies, option,
|
||||
path);
|
||||
interpreterSettingsRef.put(group, interpreterSetting);
|
||||
}
|
||||
|
|
@ -734,7 +729,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
String noteId, String key) {
|
||||
InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId);
|
||||
InterpreterOption option = interpreterSetting.getOption();
|
||||
Properties properties = interpreterSetting.getProperties();
|
||||
Properties properties = (Properties) interpreterSetting.getProperties();
|
||||
if (option.isExistingProcess) {
|
||||
properties.put(Constants.ZEPPELIN_INTERPRETER_HOST, option.getHost());
|
||||
properties.put(Constants.ZEPPELIN_INTERPRETER_PORT, option.getPort());
|
||||
|
|
@ -932,16 +927,16 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties,
|
||||
List<Dependency> dependencies) throws IOException {
|
||||
synchronized (interpreterSettings) {
|
||||
InterpreterSetting intpsetting = interpreterSettings.get(id);
|
||||
if (intpsetting != null) {
|
||||
InterpreterSetting intpSetting = interpreterSettings.get(id);
|
||||
if (intpSetting != null) {
|
||||
try {
|
||||
stopJobAllInterpreter(intpsetting);
|
||||
stopJobAllInterpreter(intpSetting);
|
||||
|
||||
intpsetting.closeAndRmoveAllInterpreterGroups();
|
||||
intpsetting.setOption(option);
|
||||
intpsetting.setProperties(properties);
|
||||
intpsetting.setDependencies(dependencies);
|
||||
loadInterpreterDependencies(intpsetting);
|
||||
intpSetting.closeAndRmoveAllInterpreterGroups();
|
||||
intpSetting.setOption(option);
|
||||
intpSetting.setProperties(properties);
|
||||
intpSetting.setDependencies(dependencies);
|
||||
loadInterpreterDependencies(intpSetting);
|
||||
|
||||
saveToFile();
|
||||
} catch (Exception e) {
|
||||
|
|
@ -960,12 +955,11 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
}
|
||||
|
||||
public void restart(String settingId, String noteId) {
|
||||
InterpreterSetting intpsetting = interpreterSettings.get(settingId);
|
||||
Preconditions.checkNotNull(intpsetting);
|
||||
InterpreterSetting intpSetting = interpreterSettings.get(settingId);
|
||||
Preconditions.checkNotNull(intpSetting);
|
||||
|
||||
if (noteIdIsExist(noteId) &&
|
||||
intpsetting.getOption().isProcess()) {
|
||||
intpsetting.closeAndRemoveInterpreterGroup(noteId);
|
||||
if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) {
|
||||
intpSetting.closeAndRemoveInterpreterGroup(noteId);
|
||||
return;
|
||||
}
|
||||
restart(settingId);
|
||||
|
|
@ -973,15 +967,15 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
|
||||
public void restart(String id) {
|
||||
synchronized (interpreterSettings) {
|
||||
InterpreterSetting intpsetting = interpreterSettings.get(id);
|
||||
InterpreterSetting intpSetting = interpreterSettings.get(id);
|
||||
// Check if dependency in specified path is changed
|
||||
// If it did, overwrite old dependency jar with new one
|
||||
if (intpsetting != null) {
|
||||
copyDependenciesFromLocalPath(intpsetting);
|
||||
if (intpSetting != null) {
|
||||
copyDependenciesFromLocalPath(intpSetting);
|
||||
|
||||
stopJobAllInterpreter(intpsetting);
|
||||
stopJobAllInterpreter(intpSetting);
|
||||
|
||||
intpsetting.closeAndRmoveAllInterpreterGroups();
|
||||
intpSetting.closeAndRmoveAllInterpreterGroups();
|
||||
|
||||
} else {
|
||||
throw new InterpreterException("Interpreter setting id " + id + " not found");
|
||||
|
|
@ -989,9 +983,9 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
}
|
||||
}
|
||||
|
||||
private void stopJobAllInterpreter(InterpreterSetting intpsetting) {
|
||||
if (intpsetting != null) {
|
||||
for (InterpreterGroup intpGroup : intpsetting.getAllInterpreterGroups()) {
|
||||
private void stopJobAllInterpreter(InterpreterSetting intpSetting) {
|
||||
if (intpSetting != null) {
|
||||
for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) {
|
||||
for (List<Interpreter> interpreters : intpGroup.values()) {
|
||||
for (Interpreter intp : interpreters) {
|
||||
for (Job job : intp.getScheduler().getJobsRunning()) {
|
||||
|
|
@ -1013,11 +1007,11 @@ public class InterpreterFactory implements InterpreterGroupFactory {
|
|||
public void close() {
|
||||
List<Thread> closeThreads = new LinkedList<>();
|
||||
synchronized (interpreterSettings) {
|
||||
Collection<InterpreterSetting> intpsettings = interpreterSettings.values();
|
||||
for (final InterpreterSetting intpsetting : intpsettings) {
|
||||
Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
|
||||
for (final InterpreterSetting intpSetting : intpSettings) {
|
||||
Thread t = new Thread() {
|
||||
public void run() {
|
||||
intpsetting.closeAndRmoveAllInterpreterGroups();
|
||||
intpSetting.closeAndRmoveAllInterpreterGroups();
|
||||
}
|
||||
};
|
||||
t.start();
|
||||
|
|
|
|||
|
|
@ -42,8 +42,19 @@ public class InterpreterSetting {
|
|||
private static final String SHARED_PROCESS = "shared_process";
|
||||
private String id;
|
||||
private String name;
|
||||
private String group; // always be null in case of InterpreterSettingRef
|
||||
private Properties properties;
|
||||
// always be null in case of InterpreterSettingRef
|
||||
private String group;
|
||||
/**
|
||||
* properties can be either Properties or Map<String, InterpreterProperty>
|
||||
* properties should be:
|
||||
* - Properties when Interpreter instances are saved to `conf/interpreter.json` file
|
||||
* - Map<String, InterpreterProperty> when Interpreters are registered
|
||||
* : this is needed after https://github.com/apache/zeppelin/pull/1145
|
||||
* which changed the way of getting default interpreter setting AKA interpreterSettingsRef
|
||||
* Note(mina): In order to simplify the implementation, I chose to change properties
|
||||
* from Properties to Object instead of creating new classes.
|
||||
*/
|
||||
private Object properties;
|
||||
private Status status;
|
||||
private String errorReason;
|
||||
|
||||
|
|
@ -65,7 +76,7 @@ public class InterpreterSetting {
|
|||
}
|
||||
|
||||
public InterpreterSetting(String id, String name, String group,
|
||||
List<InterpreterInfo> interpreterInfos, Properties properties, List<Dependency> dependencies,
|
||||
List<InterpreterInfo> interpreterInfos, Object properties, List<Dependency> dependencies,
|
||||
InterpreterOption option, String path) {
|
||||
this();
|
||||
this.id = id;
|
||||
|
|
@ -80,7 +91,7 @@ public class InterpreterSetting {
|
|||
}
|
||||
|
||||
public InterpreterSetting(String name, String group, List<InterpreterInfo> interpreterInfos,
|
||||
Properties properties, List<Dependency> dependencies, InterpreterOption option, String path) {
|
||||
Object properties, List<Dependency> dependencies, InterpreterOption option, String path) {
|
||||
this(generateId(), name, group, interpreterInfos, properties, dependencies, option, path);
|
||||
}
|
||||
|
||||
|
|
@ -174,7 +185,7 @@ public class InterpreterSetting {
|
|||
}
|
||||
}
|
||||
|
||||
public Properties getProperties() {
|
||||
public Object getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
|
|
@ -229,11 +240,7 @@ public class InterpreterSetting {
|
|||
this.option = interpreterOption;
|
||||
}
|
||||
|
||||
void updateProperties(Properties p) {
|
||||
this.properties.putAll(p);
|
||||
}
|
||||
|
||||
void setProperties(Properties p) {
|
||||
public void setProperties(Properties p) {
|
||||
this.properties = p;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.zeppelin.interpreter;
|
|||
|
||||
import java.io.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -165,12 +166,12 @@ public class InterpreterFactoryTest {
|
|||
List<String> all = factory.getDefaultInterpreterSettingList();
|
||||
// add setting with null option & properties expected nullArgumentException.class
|
||||
try {
|
||||
factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), new Properties(), "");
|
||||
factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "");
|
||||
} catch(NullArgumentException e) {
|
||||
assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
|
||||
}
|
||||
try {
|
||||
factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), new Properties(), "");
|
||||
factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "");
|
||||
} catch (NullArgumentException e){
|
||||
assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage());
|
||||
}
|
||||
|
|
@ -199,10 +200,10 @@ public class InterpreterFactoryTest {
|
|||
final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null);
|
||||
factory.add("group1", new ArrayList<InterpreterInfo>(){{
|
||||
add(info1);
|
||||
}}, new ArrayList<Dependency>(), new InterpreterOption(true), new Properties(), "/path1");
|
||||
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1");
|
||||
factory.add("group2", new ArrayList<InterpreterInfo>(){{
|
||||
add(info2);
|
||||
}}, new ArrayList<Dependency>(), new InterpreterOption(true), new Properties(), "/path2");
|
||||
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2");
|
||||
|
||||
final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
|
||||
final InterpreterSetting setting2 = factory.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties());
|
||||
|
|
@ -222,7 +223,7 @@ public class InterpreterFactoryTest {
|
|||
final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null);
|
||||
factory.add("group1", new ArrayList<InterpreterInfo>(){{
|
||||
add(info1);
|
||||
}}, new ArrayList<Dependency>(), new InterpreterOption(true), new Properties(), "/path1");
|
||||
}}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1");
|
||||
|
||||
InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED);
|
||||
final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new Properties());
|
||||
|
|
|
|||
Loading…
Reference in a new issue