Conda interpreter implementation

python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java

@@ -0,0 +1,164 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*  http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.zeppelin.python;
+
+import org.apache.zeppelin.interpreter.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Conda support
+ */
+public class PythonCondaInterpreter extends Interpreter {
+  Logger logger = LoggerFactory.getLogger(PythonCondaInterpreter.class);
+
+  Pattern condaEnvListPattern = Pattern.compile("([^\\s]*)[\\s*]*\\s(.*)");
+  Pattern listPattern = Pattern.compile("env\\s*list\\s?");
+  Pattern activatePattern = Pattern.compile("activate\\s*(.*)");
+  Pattern deactivatePattern = Pattern.compile("deactivate");
+  String pythonCommand = null;
+
+  public PythonCondaInterpreter(Properties property) {
+    super(property);
+  }
+
+  @Override
+  public void open() {
+
+  }
+
+  @Override
+  public void close() {
+
+  }
+
+  @Override
+  public InterpreterResult interpret(String st, InterpreterContext context) {
+    Matcher listMatcher = listPattern.matcher(st);
+    Matcher activateMatcher = activatePattern.matcher(st);
+    Matcher deactivateMatcher = deactivatePattern.matcher(st);
+    if (st == null || st.isEmpty() || listMatcher.matches()) {
+      return listEnv();
+    } else if (activateMatcher.matches()) {
+      String envName = activateMatcher.group(1);
+      pythonCommand = "conda run -n " + envName + " \"python -iu\"";
+      restartPythonProcess();
+      return new InterpreterResult(InterpreterResult.Code.SUCCESS, envName + " activated");
+    } else if (deactivateMatcher.matches()) {
+      pythonCommand = null;
+      restartPythonProcess();
+      return new InterpreterResult(InterpreterResult.Code.SUCCESS, "conda deactivated");
+    }
+
+    return new InterpreterResult(InterpreterResult.Code.ERROR, "Not supported command: " + st);
+  }
+
+  private void restartPythonProcess() {
+    PythonInterpreter python = getPythonInterpreter();
+    python.close();
+    python.open();
+  }
+
+  private PythonInterpreter getPythonInterpreter() {
+    LazyOpenInterpreter lazy = null;
+    PythonInterpreter python = null;
+    Interpreter p = getInterpreterInTheSameSessionByClassName(PythonInterpreter.class.getName());
+
+    while (p instanceof WrappedInterpreter) {
+      if (p instanceof LazyOpenInterpreter) {
+        lazy = (LazyOpenInterpreter) p;
+      }
+      p = ((WrappedInterpreter) p).getInnerInterpreter();
+    }
+    python = (PythonInterpreter) p;
+
+    if (lazy != null) {
+      lazy.open();
+    }
+    return python;
+  }
+
+  public String getPythonCommand() {
+    return pythonCommand;
+  }
+
+  private InterpreterResult listEnv() {
+    StringBuilder sb = new StringBuilder();
+    try {
+      int exit = runCommand(sb, "conda", "env", "list");
+      if (exit == 0) {
+        StringBuffer result = new StringBuffer();
+
+        String[] lines = sb.toString().split("\n");
+        for (String s : lines) {
+          if (s == null || s.isEmpty() || s.startsWith("#")) {
+            continue;
+          }
+          Matcher match = condaEnvListPattern.matcher(s);
+
+          if (!match.matches()) {
+            continue;
+          }
+
+          result.append(match.group(1) + "\t" + match.group(2) + "\n");
+        }
+        return new InterpreterResult(InterpreterResult.Code.SUCCESS, result.toString());
+      } else {
+        return new InterpreterResult(InterpreterResult.Code.ERROR, sb.toString());
+      }
+    } catch (IOException | InterruptedException e) {
+      throw new InterpreterException(e);
+    }
+  }
+
+
+  @Override
+  public void cancel(InterpreterContext context) {
+
+  }
+
+  @Override
+  public FormType getFormType() {
+    return FormType.NONE;
+  }
+
+  @Override
+  public int getProgress(InterpreterContext context) {
+    return 0;
+  }
+
+  private int runCommand(StringBuilder sb, String ... command)
+      throws IOException, InterruptedException {
+    ProcessBuilder builder = new ProcessBuilder(command);
+    builder.redirectErrorStream(true);
+    Process process = builder.start();
+    InputStream stdout = process.getInputStream();
+    BufferedReader br = new BufferedReader(new InputStreamReader(stdout));
+    String line;
+    while ((line = br.readLine()) != null) {
+      sb.append(line);
+      sb.append("\n");
+    }
+    int r = process.waitFor(); // Let the process finish.
+    return r;
+  }
+}

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -28,12 +28,9 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.zeppelin.display.GUI;
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.*;
 import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.scheduler.Job;
 import org.apache.zeppelin.scheduler.Scheduler;
@@ -123,6 +120,7 @@ public class PythonInterpreter extends Interpreter {
     try {
       if (process != null) {
         process.close();
+        process = null;
       }
       if (gatewayServer != null) {
         gatewayServer.shutdown();
@@ -201,7 +199,13 @@ public class PythonInterpreter extends Interpreter {
 
   public PythonProcess getPythonProcess() {
     if (process == null) {
-      return new PythonProcess(getProperty(ZEPPELIN_PYTHON));
+      PythonCondaInterpreter conda = getCondaInterpreter();
+      String binPath = getProperty(ZEPPELIN_PYTHON);
+      logger.info("CONDA " + conda.getPythonCommand());
+      if (conda.getPythonCommand() != null) {
+        binPath = conda.getPythonCommand();
+      }
+      return new PythonProcess(binPath);
     } else {
       return process;
     }
@@ -281,5 +285,25 @@ public class PythonInterpreter extends Interpreter {
   public int getMaxResult() {
     return maxResult;
   }
-  
+
+
+  private PythonCondaInterpreter getCondaInterpreter() {
+    LazyOpenInterpreter lazy = null;
+    PythonCondaInterpreter conda = null;
+    Interpreter p = getInterpreterInTheSameSessionByClassName(
+        PythonCondaInterpreter.class.getName());
+
+    while (p instanceof WrappedInterpreter) {
+      if (p instanceof LazyOpenInterpreter) {
+        lazy = (LazyOpenInterpreter) p;
+      }
+      p = ((WrappedInterpreter) p).getInnerInterpreter();
+    }
+    conda = (PythonCondaInterpreter) p;
+
+    if (lazy != null) {
+      lazy.open();
+    }
+    return conda;
+  }
 }

python/src/main/java/org/apache/zeppelin/python/PythonProcess.java

@@ -49,7 +49,23 @@ public class PythonProcess {
   }
 
   public void open() throws IOException {
-    ProcessBuilder builder = new ProcessBuilder(binPath, "-iu");
+    ProcessBuilder builder;
+    boolean hasParams = binPath.split(" ").length > 1;
+    if (System.getProperty("os.name").toLowerCase().contains("windows")) {
+      if (hasParams) {
+        builder = new ProcessBuilder(binPath.split(" "));
+      } else {
+        builder = new ProcessBuilder(binPath, "-iu");
+      }
+    } else {
+      String cmd;
+      if (hasParams) {
+        cmd = binPath;
+      } else {
+        cmd = binPath + " -iu";
+      }
+      builder = new ProcessBuilder("bash", "-c", cmd);
+    }
 
     builder.redirectErrorStream(true);
     process = builder.start();

python/src/main/resources/interpreter-setting.json

@@ -32,5 +32,16 @@
       "language": "sql",
       "editOnDblClick": false
     }
+  },
+  {
+    "group": "python",
+    "name": "conda",
+    "className": "org.apache.zeppelin.python.PythonCondaInterpreter",
+    "properties": {
+    },
+    "editor":{
+      "language": "sh",
+      "editOnDblClick": false
+    }
   }
 ]

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java

@@ -157,24 +157,25 @@ public class Paragraph extends Job implements Serializable, Cloneable {
       return null;
     }
 
+    String trimmed = text.trim();
+    if (!trimmed.startsWith("%")) {
+      return null;
+    }
+
     // get script head
     int scriptHeadIndex = 0;
-    for (int i = 0; i < text.length(); i++) {
-      char ch = text.charAt(i);
-      if (Character.isWhitespace(ch) || ch == '(') {
-        scriptHeadIndex = i;
+    for (int i = 0; i < trimmed.length(); i++) {
+      char ch = trimmed.charAt(i);
+      if (Character.isWhitespace(ch) || ch == '(' || ch == '\n') {
         break;
       }
+      scriptHeadIndex = i;
     }
-    if (scriptHeadIndex == 0) {
-      return null;
-    }
-    String head = text.substring(0, scriptHeadIndex);
-    if (head.startsWith("%")) {
-      return head.substring(1);
-    } else {
+    if (scriptHeadIndex <= 1) {
       return null;
     }
+    String head = text.substring(1, scriptHeadIndex + 1);
+    return head;
   }
 
   public String getScriptBody() {
@@ -190,10 +191,12 @@ public class Paragraph extends Job implements Serializable, Cloneable {
     if (magic == null) {
       return text;
     }
-    if (magic.length() + 1 >= text.length()) {
+
+    String trimmed = text.trim();
+    if (magic.length() + 1 >= trimmed.length()) {
       return "";
     }
-    return text.substring(magic.length() + 1).trim();
+    return trimmed.substring(magic.length() + 1).trim();
   }
 
   public Interpreter getRepl(String name) {

zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java

@@ -52,6 +52,13 @@ public class ParagraphTest {
     assertEquals(text, Paragraph.getScriptBody(text));
   }
 
+  @Test
+  public void replNameAndNoBody() {
+    String text = "%md";
+    assertEquals("md", Paragraph.getRequiredReplName(text));
+    assertEquals("", Paragraph.getScriptBody(text));
+  }
+
   @Test
   public void replNameEndsWithWhitespace() {
     String text = "%md\r\n###Hello";