Merge remote-tracking branch 'origin/master' into livyInterperter

bin/interpreter.sh

@@ -132,6 +132,14 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
 
     export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
   fi
+elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then
+  if [[ -n "${HBASE_CONF_DIR}" ]]; then
+    ZEPPELIN_CLASSPATH+=":${HBASE_CONF_DIR}"
+  elif [[ -n "${HBASE_HOME}" ]]; then
+    ZEPPELIN_CLASSPATH+=":${HBASE_HOME}/conf"
+  else
+    echo "HBASE_HOME and HBASE_CONF_DIR are not set, configuration might not be loaded"
+  fi
 fi
 
 addJarInDir "${LOCAL_INTERPRETER_REPO}"

conf/zeppelin-env.sh.template

@@ -64,3 +64,9 @@
 # export ZEPPELIN_SPARK_MAXRESULT       # Max number of SparkSQL result to display. 1000 by default.
 # export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE       # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000
 
+#### HBase interpreter configuration ####
+
+## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set
+
+# export HBASE_HOME=                    # (require) Under which HBase scripts and configuration should be
+# export HBASE_CONF_DIR=                # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml

docs/interpreter/hbase.md

@@ -10,7 +10,18 @@ group: manual
 [HBase Shell](http://hbase.apache.org/book.html#shell) is a JRuby IRB client for Apache HBase. This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin. The interpreter assumes that Apache HBase client software has been installed and it can connect to the Apache HBase cluster from the machine on where Apache Zeppelin is installed.  
 To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/book.html#quickstart)
 
-> Note: currently only HBase 1.0.x releases are supported.
+## HBase release supported
+By default, Zeppelin is built against HBase 1.0.x releases. To work with HBase 1.1.x releases, use the following build command:
+```bash
+# HBase 1.1.4
+mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.1.4 -Dhbase.hadoop.version=2.6.0
+```
+To work with HBase 1.2.0+, use the following build command:
+
+```bash
+# HBase 1.2.0
+mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.2.0 -Dhbase.hadoop.version=2.6.0
+```
 
 ## Configuration
 
@@ -23,7 +34,7 @@ To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/
   <tr>
     <td>hbase.home</td>
     <td>/usr/lib/hbase</td>
-    <td>Installation directory of Hbase</td>
+    <td>Installation directory of HBase, defaults to HBASE_HOME in environment</td>
   </tr>
   <tr>
     <td>hbase.ruby.sources</td>
@@ -31,12 +42,31 @@ To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/
     <td>Path to Ruby scripts relative to 'hbase.home'</td>
   </tr>
   <tr>
-    <td>hbase.test.mode</td>
+    <td>zeppelin.hbase.test.mode</td>
     <td>false</td>
     <td>Disable checks for unit and manual tests</td>
   </tr>
 </table>
 
+If you want to connect to HBase running on a cluster, you'll need to follow the next step.
+
+### Export HBASE_HOME
+In **conf/zeppelin-env.sh**, export `HBASE_HOME` environment variable with your HBase installation path. This ensures `hbase-site.xml` can be loaded.
+
+for example
+
+```bash
+export HBASE_HOME=/usr/lib/hbase
+```
+
+or, when running with CDH
+
+```bash
+export HBASE_HOME="/opt/cloudera/parcels/CDH/lib/hbase"
+```
+
+You can optionally export `HBASE_CONF_DIR` instead of `HBASE_HOME` should you have custom HBase configurations.
+
 ## Enabling the HBase Shell Interpreter
 
 In a notebook, to enable the **HBase Shell** interpreter, click the **Gear** icon and select **HBase Shell**.

flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java

@@ -249,12 +249,34 @@ public class FlinkInterpreter extends Interpreter {
     Code r = null;
 
     String incomplete = "";
+    boolean inComment = false;
+
     for (int l = 0; l < linesToRun.length; l++) {
       final String s = linesToRun[l];
       // check if next line starts with "." (but not ".." or "./") it is treated as an invocation
       if (l + 1 < linesToRun.length) {
         String nextLine = linesToRun[l + 1].trim();
-        if (nextLine.startsWith(".") && !nextLine.startsWith("..") && !nextLine.startsWith("./")) {
+        boolean continuation = false;
+        if (nextLine.isEmpty()
+                || nextLine.startsWith("//")         // skip empty line or comment
+                || nextLine.startsWith("}")
+                || nextLine.startsWith("object")) { // include "} object" for Scala companion object
+          continuation = true;
+        } else if (!inComment && nextLine.startsWith("/*")) {
+          inComment = true;
+          continuation = true;
+        } else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
+          inComment = false;
+          continuation = true;
+        } else if (nextLine.length() > 1
+                && nextLine.charAt(0) == '.'
+                && nextLine.charAt(1) != '.'     // ".."
+                && nextLine.charAt(1) != '/') {  // "./"
+          continuation = true;
+        } else if (inComment) {
+          continuation = true;
+        }
+        if (continuation) {
           incomplete += s + "\n";
           continue;
         }

flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java

@@ -49,6 +49,22 @@ public class FlinkInterpreterTest {
     flink.destroy();
   }
 
+  @Test
+  public void testNextLineInvocation() {
+    assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n.toInt", context).code());
+  }
+
+  @Test
+  public void testNextLineComments() {
+    assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n/*comment here\n*/.toInt", context).code());
+  }
+
+  @Test
+  public void testNextLineCompanionObject() {
+    String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}";
+    assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret(code, context).code());
+  }
+
   @Test
   public void testSimpleStatement() {
     InterpreterResult result = flink.interpret("val a=1", context);
@@ -63,12 +79,6 @@ public class FlinkInterpreterTest {
     assertEquals("1", result.message());
   }
 
-  @Test
-  public void testNextlineInvoke() {
-    InterpreterResult result = flink.interpret("\"123\"\n  .toInt", context);
-    assertEquals("res0: Int = 123\n", result.message());    
-  }
-
   @Test
   public void testWordCount() {
     flink.interpret("val text = env.fromElements(\"To be or not to be\")", context);

hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java

@@ -37,21 +37,20 @@ import java.util.List;
 import java.util.Properties;
 
 /**
- * Support for Hbase Shell. All the commands documented here
+ * Support for HBase Shell. All the commands documented here
  * http://hbase.apache.org/book.html#shell is supported.
  *
  * Requirements:
- * Hbase Shell should be installed on the same machine. To be more specific, the following dir.
+ * HBase Shell should be installed on the same machine. To be more specific, the following dir.
  * should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby
- * Hbase Shell should be able to connect to the Hbase cluster from terminal. This makes sure
+ * HBase Shell should be able to connect to the HBase cluster from terminal. This makes sure
  * that the client is configured properly.
  *
  * The interpreter takes 3 config parameters:
- * hbase.home: Root dir. where hbase is installed. Default is /usr/lib/hbase/
+ * hbase.home: Root directory where HBase is installed. Default is /usr/lib/hbase/
  * hbase.ruby.sources: Dir where shell ruby code is installed.
  *                          Path is relative to hbase.home. Default: lib/ruby
- * hbase.irb.load: (Testing only) Default is true.
- *                      Whether to load irb in the interpreter.
+ * zeppelin.hbase.test.mode: (Testing only) Disable checks for unit and manual tests. Default: false
  */
 public class HbaseInterpreter extends Interpreter {
   private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class);
@@ -62,11 +61,13 @@ public class HbaseInterpreter extends Interpreter {
   static {
     Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(),
         new InterpreterPropertyBuilder()
-            .add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase")
+            .add("hbase.home",
+              getSystemDefault("HBASE_HOME", "hbase.home", "/usr/lib/hbase/"),
+              "Installation directory of HBase")
             .add("hbase.ruby.sources", "lib/ruby",
                 "Path to Ruby scripts relative to 'hbase.home'")
-            .add("hbase.test.mode", "false", "Disable checks for unit and manual tests")
-            .build());
+            .add("zeppelin.hbase.test.mode", "false", "Disable checks for unit and manual tests")
+          .build());
   }
 
   public HbaseInterpreter(Properties property) {
@@ -79,7 +80,7 @@ public class HbaseInterpreter extends Interpreter {
     this.writer = new StringWriter();
     scriptingContainer.setOutput(this.writer);
 
-    if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) {
+    if (!Boolean.parseBoolean(getProperty("zeppelin.hbase.test.mode"))) {
       String hbase_home = getProperty("hbase.home");
       String ruby_src = getProperty("hbase.ruby.sources");
       Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath();
@@ -89,7 +90,7 @@ public class HbaseInterpreter extends Interpreter {
 
       File f = abs_ruby_src.toFile();
       if (!f.exists() || !f.isDirectory()) {
-        throw new InterpreterException("hbase ruby sources is not available at '" + abs_ruby_src
+        throw new InterpreterException("HBase ruby sources is not available at '" + abs_ruby_src
             + "'");
       }
 
@@ -155,4 +156,24 @@ public class HbaseInterpreter extends Interpreter {
     return null;
   }
 
+  private static String getSystemDefault(
+      String envName,
+      String propertyName,
+      String defaultValue) {
+
+    if (envName != null && !envName.isEmpty()) {
+      String envValue = System.getenv().get(envName);
+      if (envValue != null) {
+        return envValue;
+      }
+    }
+
+    if (propertyName != null && !propertyName.isEmpty()) {
+      String propValue = System.getProperty(propertyName);
+      if (propValue != null) {
+        return propValue;
+      }
+    }
+    return defaultValue;
+  }
 }

hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java

@@ -40,7 +40,7 @@ public class HbaseInterpreterTest {
     Properties properties = new Properties();
     properties.put("hbase.home", "");
     properties.put("hbase.ruby.sources", "");
-    properties.put("hbase.test.mode", "true");
+    properties.put("zeppelin.hbase.test.mode", "true");
 
     hbaseInterpreter = new HbaseInterpreter(properties);
     hbaseInterpreter.open();
@@ -72,4 +72,4 @@ public class HbaseInterpreterTest {
     assertEquals(InterpreterResult.Code.ERROR, result.code());
     assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message());
   }
-}
+}

scalding/src/main/java/org/apache/zeppelin/scalding/ScaldingInterpreter.java

@@ -209,13 +209,34 @@ public class ScaldingInterpreter extends Interpreter {
     out.reset();
     Code r = null;
     String incomplete = "";
+    boolean inComment = false;
 
     for (int l = 0; l < linesToRun.length; l++) {
       String s = linesToRun[l];
       // check if next line starts with "." (but not ".." or "./") it is treated as an invocation
       if (l + 1 < linesToRun.length) {
         String nextLine = linesToRun[l + 1].trim();
-        if (nextLine.startsWith(".") && !nextLine.startsWith("..") && !nextLine.startsWith("./")) {
+        boolean continuation = false;
+        if (nextLine.isEmpty()
+                || nextLine.startsWith("//")         // skip empty line or comment
+                || nextLine.startsWith("}")
+                || nextLine.startsWith("object")) { // include "} object" for Scala companion object
+          continuation = true;
+        } else if (!inComment && nextLine.startsWith("/*")) {
+          inComment = true;
+          continuation = true;
+        } else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
+          inComment = false;
+          continuation = true;
+        } else if (nextLine.length() > 1
+                && nextLine.charAt(0) == '.'
+                && nextLine.charAt(1) != '.'     // ".."
+                && nextLine.charAt(1) != '/') {  // "./"
+          continuation = true;
+        } else if (inComment) {
+          continuation = true;
+        }
+        if (continuation) {
           incomplete += s + "\n";
           continue;
         }

scalding/src/test/java/org/apache/zeppelin/scalding/ScaldingInterpreterTest.java

@@ -88,6 +88,17 @@ public class ScaldingInterpreterTest {
     }
   }
 
+  @Test
+  public void testNextLineComments() {
+    assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("\"123\"\n/*comment here\n*/.toInt", context).code());
+  }
+
+  @Test
+  public void testNextLineCompanionObject() {
+    String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}";
+    assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret(code, context).code());
+  }
+
   @Test
   public void testBasicIntp() {
     assertEquals(InterpreterResult.Code.SUCCESS,

zeppelin-web/src/app/notebook/paragraph/paragraph.css

@@ -48,10 +48,6 @@
   margin-bottom: 0;
 }
 
-.paragraph p {
-  margin: 0;
-}
-
 .paragraph div svg {
   width: 100%;
 }