Merge branch 'master' into ZEPPELIN-732-up

docs/index.md

@@ -33,7 +33,7 @@ limitations under the License.
     </ul>
   </div>
   <div class="col-md-6" style="padding:0">
-    <img class="img-responsive" style="border: 1px solid #ecf0f1;" src="/assets/themes/zeppelin/img/notebook.png" />
+    <img class="img-responsive" style="border: 1px solid #ecf0f1;" src="../assets/themes/zeppelin/img/notebook.png" />
   </div>
 </div>
 
@@ -42,14 +42,14 @@ limitations under the License.
 [Apache Zeppelin interpreter](./manual/interpreters.html) concept allows any language/data-processing-backend to be plugged into Zeppelin.
 Currently Apache Zeppelin supports many interpreters such as Apache Spark, Python, JDBC, Markdown and Shell.
 
-<img class="img-responsive" width="500px" style="margin:0 auto; padding: 26px;" src="/assets/themes/zeppelin/img/available_interpreters.png" />
+<img class="img-responsive" width="500px" style="margin:0 auto; padding: 26px;" src="../assets/themes/zeppelin/img/available_interpreters.png" />
 
 Adding new language-backend is really simple. Learn [how to create your own interpreter](./development/writingzeppelininterpreter.html#make-your-own-interpreter).
 
 #### Apache Spark integration
 Especially, Apache Zeppelin provides built-in [Apache Spark](http://spark.apache.org/) integration. You don't need to build a separate module, plugin or library for it.
 
-<img class="img-responsive" src="/assets/themes/zeppelin/img/spark_logo.jpg" width="140px" />
+<img class="img-responsive" src="../assets/themes/zeppelin/img/spark_logo.jpg" width="140px" />
 
 Apache Zeppelin with Spark integration provides
 
@@ -66,10 +66,10 @@ Some basic charts are already included in Apache Zeppelin. Visualizations are no
 
 <div class="row">
   <div class="col-md-6">
-    <img class="img-responsive" src="/assets/themes/zeppelin/img/graph1.png" />
+    <img class="img-responsive" src="../assets/themes/zeppelin/img/graph1.png" />
   </div>
   <div class="col-md-6">
-    <img class="img-responsive" src="/assets/themes/zeppelin/img/graph2.png" />
+    <img class="img-responsive" src="../assets/themes/zeppelin/img/graph2.png" />
   </div>
 </div>
 
@@ -79,7 +79,7 @@ Apache Zeppelin aggregates values and displays them in pivot chart with simple d
 
 <div class="row">
   <div class="col-md-12">
-    <img class="img-responsive" style="margin: 16px auto;" src="/assets/themes/zeppelin/img/screenshots/pivot.png" width="480px" />
+    <img class="img-responsive" style="margin: 16px auto;" src="../assets/themes/zeppelin/img/screenshots/pivot.png" width="480px" />
   </div>
 </div>
 
@@ -91,7 +91,7 @@ Learn more about [display systems](#display-system) in Apache Zeppelin.
 Apache Zeppelin can dynamically create some input forms in your notebook.
 <div class="row">
   <div class="col-md-12">
-    <img class="img-responsive" style="margin: 16px auto;" src="/assets/themes/zeppelin/img/screenshots/dynamicform.png" />
+    <img class="img-responsive" style="margin: 16px auto;" src="../assets/themes/zeppelin/img/screenshots/dynamicform.png" />
   </div>
 </div>
 Learn more about [Dynamic Forms](./manual/dynamicform.html).
@@ -102,7 +102,7 @@ Your notebook URL can be shared among collaborators. Then Apache Zeppelin will b
 
 <div class="row">
   <div class="col-md-12">
-    <img class="img-responsive" style="margin: 20px auto" src="/assets/themes/zeppelin/img/screenshots/publish.png" width="650px"/>
+    <img class="img-responsive" style="margin: 20px auto" src="../assets/themes/zeppelin/img/screenshots/publish.png" width="650px"/>
   </div>
 </div>
 
@@ -113,7 +113,7 @@ If you want to learn more about this feature, please visit [this page](./manual/
 <br />
 ## 100% Opensource
 
-<img class="img-responsive" style="margin:0 auto; padding: 15px;" src="/assets/themes/zeppelin/img/asf_logo.png" width="250px"/>
+<img class="img-responsive" style="margin:0 auto; padding: 15px;" src="../assets/themes/zeppelin/img/asf_logo.png" width="250px"/>
 
 Apache Zeppelin is Apache2 Licensed software. Please check out the [source repository](http://git.apache.org/zeppelin.git) and [how to contribute](./development/howtocontribute.html).
 Apache Zeppelin has a very active development community.

zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java

@@ -71,7 +71,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
    *url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
    * @throws ConfigurationException
    */
-  public static ZeppelinConfiguration create() {
+  public static synchronized ZeppelinConfiguration create() {
     if (conf != null) {
       return conf;
     }
@@ -423,6 +423,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     return getString(ConfVars.ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
   }
 
+  public boolean getUseJdbcAlias() {
+    return getBoolean(ConfVars.ZEPPELIN_USE_JDBC_ALIAS);
+  }
+
   public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
                                                 ConfigurationKeyPredicate predicate) {
     Map<String, String> configurations = new HashMap<>();
@@ -544,7 +548,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
     ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true),
     ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true),
-    ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000");
+    ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"),
+    ZEPPELIN_USE_JDBC_ALIAS("zeppelin.use.jdbc.alias", true);
 
 
     private String varName;

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java

@@ -25,6 +25,7 @@ import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.Input;
@@ -64,6 +65,8 @@ public class Note implements Serializable, ParagraphJobListener {
   private String name = "";
   private String id;
 
+  private transient ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+
   @SuppressWarnings("rawtypes")
   Map<String, List<AngularObject>> angularObjects = new HashMap<>();
 
@@ -421,10 +424,20 @@ public class Note implements Serializable, ParagraphJobListener {
   public void run(String paragraphId) {
     Paragraph p = getParagraph(paragraphId);
     p.setNoteReplLoader(replLoader);
+
     p.setListener(this);
-    Interpreter intp = replLoader.get(p.getRequiredReplName());
+    String requiredReplName = p.getRequiredReplName();
+    Interpreter intp = replLoader.get(requiredReplName);
+
     if (intp == null) {
-      throw new InterpreterException("Interpreter " + p.getRequiredReplName() + " not found");
+      // TODO(jongyoul): Make "%jdbc" configurable from JdbcInterpreter
+      if (conf.getUseJdbcAlias() && null != (intp = replLoader.get("jdbc"))) {
+        String pText = p.getText().replaceFirst(requiredReplName, "jdbc(" + requiredReplName + ")");
+        logger.debug("New paragraph: {}", pText);
+        p.setEffectiveText(pText);
+      } else {
+        throw new InterpreterException("Interpreter " + requiredReplName + " not found");
+      }
     }
     if (p.getConfig().get("enabled") == null || (Boolean) p.getConfig().get("enabled")) {
       intp.getScheduler().submit(p);

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java

@@ -194,6 +194,6 @@ public class NoteInterpreterLoader {
       return factory.getDevInterpreter();
     }
 
-    throw new InterpreterException(replName + " interpreter not found");
+    return null;
   }
 }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java

@@ -52,6 +52,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
   private transient NoteInterpreterLoader replLoader;
   private transient Note note;
   private transient AuthenticationInfo authenticationInfo;
+  private transient String effectiveText;
 
   String title;
   String text;
@@ -112,6 +113,14 @@ public class Paragraph extends Job implements Serializable, Cloneable {
     this.dateUpdated = new Date();
   }
 
+  public void setEffectiveText(String effectiveText) {
+    this.effectiveText = effectiveText;
+  }
+
+  public String getEffectiveText() {
+    return effectiveText;
+  }
+
   public AuthenticationInfo getAuthenticationInfo() {
     return authenticationInfo;
   }
@@ -143,7 +152,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
   }
 
   public String getRequiredReplName() {
-    return getRequiredReplName(text);
+    return getRequiredReplName(null != effectiveText ? effectiveText : text);
   }
 
   public static String getRequiredReplName(String text) {
@@ -171,8 +180,8 @@ public class Paragraph extends Job implements Serializable, Cloneable {
     }
   }
 
-  private String getScriptBody() {
-    return getScriptBody(text);
+  public String getScriptBody() {
+    return getScriptBody(null != effectiveText ? effectiveText : text);
   }
 
   public static String getScriptBody(String text) {
@@ -303,6 +312,7 @@ public class Paragraph extends Job implements Serializable, Cloneable {
       }
     } finally {
       InterpreterContext.remove();
+      effectiveText = null;
     }
   }
 

zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook;
+
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.notebook.repo.NotebookRepo;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.search.SearchService;
+import org.apache.zeppelin.user.Credentials;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+@RunWith(MockitoJUnitRunner.class)
+public class NoteTest {
+  @Mock
+  NotebookRepo repo;
+
+  @Mock
+  NoteInterpreterLoader replLoader;
+
+  @Mock
+  JobListenerFactory jobListenerFactory;
+
+  @Mock
+  SearchService index;
+
+  @Mock
+  Credentials credentials;
+
+  @Mock
+  Interpreter interpreter;
+
+  @Mock
+  Scheduler scheduler;
+
+  @Test
+  public void runNormalTest() {
+    when(replLoader.get("spark")).thenReturn(interpreter);
+    when(interpreter.getScheduler()).thenReturn(scheduler);
+
+    String pText = "%spark sc.version";
+    Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials);
+    Paragraph p = note.addParagraph();
+    p.setText(pText);
+    note.run(p.getId());
+
+    ArgumentCaptor<Paragraph> pCaptor = ArgumentCaptor.forClass(Paragraph.class);
+    verify(scheduler, only()).submit(pCaptor.capture());
+    verify(replLoader, only()).get("spark");
+
+    assertEquals("Paragraph text", pText, pCaptor.getValue().getText());
+  }
+
+  @Test
+  public void runJdbcTest() {
+    when(replLoader.get("mysql")).thenReturn(null);
+    when(replLoader.get("jdbc")).thenReturn(interpreter);
+    when(interpreter.getScheduler()).thenReturn(scheduler);
+
+    String pText = "%mysql show databases";
+    Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials);
+    Paragraph p = note.addParagraph();
+    p.setText(pText);
+    note.run(p.getId());
+
+    ArgumentCaptor<Paragraph> pCaptor = ArgumentCaptor.forClass(Paragraph.class);
+    verify(scheduler, only()).submit(pCaptor.capture());
+    verify(replLoader, times(2)).get(anyString());
+
+    assertEquals("Change paragraph text", "%jdbc(mysql) show databases", pCaptor.getValue().getEffectiveText());
+    assertEquals("Change paragraph text", pText, pCaptor.getValue().getText());
+  }
+}

zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java

@@ -27,6 +27,7 @@ import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectBuilder;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.display.Input;
+import org.apache.zeppelin.interpreter.Interpreter;
 import org.junit.Test;
 
 import java.util.HashMap;
@@ -69,6 +70,30 @@ public class ParagraphTest {
     assertEquals("md", Paragraph.getRequiredReplName(text));
   }
 
+  @Test
+  public void effectiveTextTest() {
+    NoteInterpreterLoader noteInterpreterLoader = mock(NoteInterpreterLoader.class);
+    Interpreter interpreter = mock(Interpreter.class);
+
+    Paragraph p = new Paragraph(null, null, null, noteInterpreterLoader);
+    p.setText("%h2 show databases");
+    p.setEffectiveText("%jdbc(h2) show databases");
+    assertEquals("Get right replName", "jdbc", p.getRequiredReplName());
+    assertEquals("Get right scriptBody", "(h2) show databases", p.getScriptBody());
+
+    when(noteInterpreterLoader.get("jdbc")).thenReturn(interpreter);
+    when(interpreter.getFormType()).thenReturn(Interpreter.FormType.NATIVE);
+
+    try {
+      p.jobRun();
+    } catch (Throwable throwable) {
+      // Do nothing
+    }
+
+    assertEquals("Erase effective Text", "h2", p.getRequiredReplName());
+    assertEquals("Erase effective Text", "show databases", p.getScriptBody());
+  }
+
   @Test
   public void should_extract_variable_from_angular_object_registry() throws Exception {
     //Given