Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

make spark container image configurable

Browse source
This commit is contained in:
Lee moon soo 2018-11-23 13:31:12 +09:00
parent 0d472ea522
commit f4166ad04c
5 changed files with 15 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
k8s/interpreter/100-interpreter-pod.yaml
View file

@ -39,7 +39,7 @@ spec:
mountPath: /spark
initContainers:
- name: spark-home-init
image: {{zeppelin.k8s.spark.image}}
image: {{zeppelin.k8s.spark.container.image}}
command: ["sh", "-c", "cp -r /opt/spark/* /spark/"]
volumeMounts:
- name: spark-home

2
k8s/zeppelin.yaml
View file

@ -26,6 +26,8 @@ spec:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE
value: spark:2.4.0
---
kind: Service
apiVersion: v1

9
zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -681,6 +681,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_K8S_CONTAINER_IMAGE);
}
public String getK8sSparkContainerImage() {
return getString(ConfVars.ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE);
}
public String getK8sTemplatesDir() {
return getRelativeDir(ConfVars.ZEPPELIN_K8S_TEMPLATE_DIR);
}
@ -836,10 +840,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_CLUSTER_HEARTBEAT_INTERVAL("zeppelin.cluster.heartbeat.interval", 3000),
ZEPPELIN_CLUSTER_HEARTBEAT_TIMEOUT("zeppelin.cluster.heartbeat.timeout", 9000),
ZEPPELIN_K8S_MODE("zeppelin.k8s.mode", "auto"), // auto | on | off
ZEPPELIN_K8S_MODE("zeppelin.k8s.mode", "auto"), // auto | on | off
ZEPPELIN_K8S_PORTFORWARD("zeppelin.k8s.portforward", false), // kubectl port-forward incase of Zeppelin is running outside of kuberentes
ZEPPELIN_K8S_KUBECTL("zeppelin.k8s.kubectl", "kubectl"), // kubectl command
ZEPPELIN_K8S_KUBECTL("zeppelin.k8s.kubectl", "kubectl"), // kubectl command
ZEPPELIN_K8S_CONTAINER_IMAGE("zeppelin.k8s.container.image", "apache/zeppelin:" + Util.getVersion()),
ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE("zeppelin.k8s.spark.container.image", "apache/spark:latest"),
ZEPPELIN_K8S_TEMPLATE_DIR("zeppelin.k8s.template.dir", "k8s"),
ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL("zeppelin.notebook.git.remote.url", ""),

5
zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java
View file

@ -35,6 +35,7 @@ public class K8sRemoteInterpreterProcess extends RemoteInterpreterProcess {
private final Gson gson = new Gson();
private final String podName;
private final boolean portForward;
private final String sparkImage;
private ExecuteWatchdog portForwardWatchdog;
private int podPort = K8S_INTERPRETER_SERVICE_PORT;
@ -52,6 +53,7 @@ public class K8sRemoteInterpreterProcess extends RemoteInterpreterProcess {
String zeppelinServiceHost,
String zeppelinServiceRpcPort,
boolean portForward,
String sparkImage,
int connectTimeout) {
super(connectTimeout);
this.kubectl = kubectl;
@ -65,6 +67,7 @@ public class K8sRemoteInterpreterProcess extends RemoteInterpreterProcess {
this.zeppelinServiceHost = zeppelinServiceHost;
this.zeppelinServiceRpcPort = zeppelinServiceRpcPort;
this.portForward = portForward;
this.sparkImage = sparkImage;
this.podName = interpreterGroupName.toLowerCase() + "-" + getRandomString(6);
}
@ -258,7 +261,7 @@ public class K8sRemoteInterpreterProcess extends RemoteInterpreterProcess {
envs.put("ZEPPELIN_HOME", envs.getOrDefault("ZEPPELIN_HOME", "/zeppelin"));
if (isSpark()) {
k8sProperties.put("zeppelin.k8s.spark.image", "spark:2.4.0");
k8sProperties.put("zeppelin.k8s.spark.container.image", sparkImage);
envs.put("SPARK_SUBMIT_OPTIONS", envs.getOrDefault("SPARK_SUBMIT_OPTIONS", "") + buildSparkSubmitOptions());
envs.put("SPARK_HOME", envs.getOrDefault("SPARK_HOME", "/spark"));
}

1
zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sStandardInterpreterLauncher.java
View file

@ -134,6 +134,7 @@ public class K8sStandardInterpreterLauncher extends InterpreterLauncher {
getZeppelinServiceHost(),
getZeppelinServiceRpcPort(),
zConf.getK8sPortForward(),
zConf.getK8sSparkContainerImage(),
connectTimeout);
}