spark ui support

k8s/interpreter/100-interpreter-pod.yaml

@@ -89,6 +89,8 @@ spec:
       port: 22321
     - name: spark-blockmanager
       port: 22322
+    - name: spark-ui
+      port: 4040
     {% endif %}
   selector:
     app: {{zeppelin.k8s.interpreter.pod.name}}

k8s/zeppelin-server.yaml

@@ -14,6 +14,58 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: zeppelin-server-conf
+data:
+  serviceDomain: localtest.me:8080
+  sparkContainerImage: spark:2.4.0
+  nginx.conf: |
+    daemon off;
+    worker_processes auto;
+    events {
+      worker_connections 1024;
+    }
+    http {
+      map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+      }
+
+      # first server block will be default. Proxy zeppelin server.
+      server {
+        listen 80;
+        location / {
+          proxy_pass http://localhost:8080;
+          proxy_set_header Host $host;
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection $connection_upgrade;
+          proxy_redirect http://localhost $scheme://SERVICE_DOMAIN;
+        }
+      }
+
+      # match request domain [port]-[service].[serviceDomain]
+      # proxy extra service such as spark-ui
+      server {
+        listen 80;
+        server_name "~(?<svc_port>[0-9]+)-(?<svc_name>[^.]*)\.(.*)";
+        location / {
+          resolver 127.0.0.1:53 ipv6=off;
+          proxy_pass http://$svc_name.NAMESPACE.svc.cluster.local:$svc_port;
+          proxy_set_header Host $host;
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection $connection_upgrade;
+          proxy_redirect http://localhost $scheme://SERVICE_DOMAIN;
+
+          # redirect rule for spark ui. 302 redirect response misses port number of service domain
+          proxy_redirect ~(http:[/]+[0-9]+[-][^-]+[-][^.]+)[^/]+(\/jobs.*) $1.SERVICE_DOMAIN$2;
+        }
+      }
+    }
+---
 kind: Pod
 apiVersion: v1
 metadata:
@@ -42,7 +94,50 @@ spec:
           apiVersion: v1
           fieldPath: metadata.name
     - name: ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE
-      value: spark:2.4.0
+      valueFrom:
+        configMapKeyRef:
+          name: zeppelin-server-conf
+          key: sparkContainerImage
+    - name: SERVICE_DOMAIN
+      valueFrom:
+        configMapKeyRef:
+          name: zeppelin-server-conf
+          key: serviceDomain
+  - name: zeppelin-server-gateway
+    image: nginx:1.14.0
+    command: [ "/bin/sh", "-c" ]
+    args:
+      - cp -f /tmp/conf/nginx.conf /etc/nginx/nginx.conf;
+        sed -i -e "s/SERVICE_DOMAIN/$(cat /tmp/conf/serviceDomain)/g" /etc/nginx/nginx.conf;
+        sed -i -e "s/NAMESPACE/$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/g" /etc/nginx/nginx.conf;
+        cat /etc/nginx/nginx.conf;
+        /usr/sbin/nginx
+    volumeMounts:
+      - name: zeppelin-server-conf-volume
+        mountPath: /tmp/conf
+    lifecycle:
+      preStop:
+        exec:
+          # SIGTERM triggers a quick exit; gracefully terminate instead
+          command: ["/usr/sbin/nginx","-s","quit"]
+  - name: dnsmasq
+    image: "janeczku/go-dnsmasq:release-1.0.5"
+    args:
+      - --listen
+      - "127.0.0.1:53"
+      - --default-resolver
+      - --append-search-domains
+      - --hostsfile=/etc/hosts
+      - --verbose
+  volumes:
+  - name: zeppelin-server-conf-volume
+    configMap:
+      name: zeppelin-server-conf
+      items:
+      - key: nginx.conf
+        path: nginx.conf
+      - key: serviceDomain
+        path: serviceDomain
 ---
 kind: Service
 apiVersion: v1
@@ -51,7 +146,7 @@ metadata:
 spec:
   ports:
     - name: http
-      port: 8080
+      port: 80
     - name: rpc            # port name is referenced in the code. So it shouldn't be changed.
       port: 12320
   selector:

zeppelin-plugins/launcher/k8s-standard/src/main/java/org/apache/zeppelin/interpreter/launcher/K8sRemoteInterpreterProcess.java

@@ -258,12 +258,21 @@ public class K8sRemoteInterpreterProcess extends RemoteInterpreterProcess {
     }
 
     // environment variables
+    envs.put("SERVICE_DOMAIN", envs.getOrDefault("SERVICE_DOMAIN", System.getenv("SERVICE_DOMAIN")));
     envs.put("ZEPPELIN_HOME", envs.getOrDefault("ZEPPELIN_HOME", "/zeppelin"));
 
     if (isSpark()) {
+      int webUiPort = 4040;
       k8sProperties.put("zeppelin.k8s.spark.container.image", sparkImage);
       envs.put("SPARK_SUBMIT_OPTIONS", envs.getOrDefault("SPARK_SUBMIT_OPTIONS", "") + buildSparkSubmitOptions());
       envs.put("SPARK_HOME", envs.getOrDefault("SPARK_HOME", "/spark"));
+
+      // configure interpreter property "zeppelin.spark.uiWebUrl" if not defined, to enable spark ui through reverse proxy
+      String webUrl = (String) properties.get("zeppelin.spark.uiWebUrl");
+      if (webUrl == null || webUrl.trim().isEmpty()) {
+        properties.put("zeppelin.spark.uiWebUrl",
+            String.format("//%d-%s.%s", webUiPort, getPodName(), envs.get("SERVICE_DOMAIN")));
+      }
     }
 
     k8sProperties.put("zeppelin.k8s.envs", envs);
@@ -273,8 +282,6 @@ public class K8sRemoteInterpreterProcess extends RemoteInterpreterProcess {
     return k8sProperties;
   }
 
-
-
   private boolean isSpark() {
     return "spark".equalsIgnoreCase(interpreterGroupName);
   }