Merge branch 'master' of https://github.com/apache/incubator-zeppelin

bin/common.sh

@@ -105,6 +105,7 @@ if [[ -z "$ZEPPELIN_MEM" ]]; then
 fi
 
 JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM}"
+JAVA_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
 export JAVA_OPTS
 
 # jvm options for interpreter process
@@ -117,6 +118,7 @@ if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then
 fi
 
 JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}"
+JAVA_INTP_OPTS+=" -Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j.properties"
 export JAVA_INTP_OPTS
 
 

docs/install/virtual_machine.md

@@ -26,11 +26,13 @@ Apache Zeppelin distribution includes a scripts directory
 
  `scripts/vagrant/zeppelin-dev`
  
-This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin.  It can also be used to run an existing Zeppelin build if you don't plan to build from source.  For pyspark users, this script also includes several helpful [Python Libraries](#pythonextras).
+This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin.  It can also be used to run an existing Zeppelin build if you don't plan to build from source.
+For PySpark users, this script includes several helpful [Python Libraries](#python-extras).
+For SparkR users, this script includes several helpful [R Libraries](#r-extras).
  
 ####Installing the required components to launch a virtual machine.
 
-This script requires three applications, [Ansible](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip "Ansible"), [Vagrant](http://www.vagrantup.com/downloads "Vagrant") and [Virtual Box](https://www.virtualbox.org/ "Virtual Box").  All of these applications are freely available as Open Source projects and extremely easy to set up on most operating systems.
+This script requires three applications, [Ansible](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip "Ansible"), [Vagrant](http://www.vagrantup.com "Vagrant") and [Virtual Box](https://www.virtualbox.org/ "Virtual Box").  All of these applications are freely available as Open Source projects and extremely easy to set up on most operating systems.
 
 ### Create a Zeppelin Ready VM in 4 Steps (5 on Windows)
 
@@ -92,23 +94,24 @@ The virtual machine consists of:
  - libfontconfig to avoid phatomJs missing dependency issues
  - openjdk-7-jdk
  - Python addons: pip, matplotlib, scipy, numpy, pandas
+ - [R](https://www.r-project.org/) and R Packages required to run the R Interpreter and the related R tutorial notebook, including:  Knitr, devtools, repr, rCharts, ggplot2, googleVis, mplot, htmltools, base64enc, data.table
  
 ### How to build & run Zeppelin
 
-This assumes you've already cloned the project either on the host machine in the zeppelin-dev directory (to be shared with the guest machine) or cloned directly into a directory while running inside the guest machine.
+This assumes you've already cloned the project either on the host machine in the zeppelin-dev directory (to be shared with the guest machine) or cloned directly into a directory while running inside the guest machine.  The following build steps will also include Python and R support via PySpark and SparkR:
 
 ```
 cd /incubator-zeppelin
-mvn clean package -Pspark-1.5 -Ppyspark -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
+mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests
 ./bin/zeppelin-daemon.sh start
 ```
 
 On your host machine browse to `http://localhost:8080/`
 
-If you [turned off port forwarding](#tweakvm) in the `Vagrantfile` browse to `http://192.168.51.52:8080`
+If you [turned off port forwarding](#tweaking-the-virtual-machine) in the `Vagrantfile` browse to `http://192.168.51.52:8080`
 
 
-### [Tweaking the Virtual Machine](id:tweakvm)
+### Tweaking the Virtual Machine
 
 If you plan to run this virtual machine along side other Vagrant images, you may wish to bind the virtual machine to a specific IP address, and not use port fowarding from your local host.
 
@@ -123,7 +126,7 @@ config.vm.network "private_network", ip: "192.168.51.52"
 This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources.  You may wish to launch nodes in virtual machines with IP addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
 
 
-### [Python Extras](id:pythonextras)
+### Python Extras
 
 With Zeppelin running, **Numpy**, **SciPy**, **Pandas** and **Matplotlib** will be available.  Create a pyspark notebook, and try the below code.
 
@@ -175,10 +178,7 @@ plt.title('How fast do you want to go today?')
 show(plt)
 ``` 
 
+### R Extras
 
-
-
-
-
-
-
+With zeppelin running, an R Tutorial notebook will be available.  The R packages required to run the examples and graphs in this tutorial notebook were installed by this virtual machine.
+The installed R Packages include: Knitr, devtools, repr, rCharts, ggplot2, googleVis, mplot, htmltools, base64enc, data.table

docs/manual/interpreters.md

@@ -36,6 +36,8 @@ Zeppelin interpreter setting is the configuration of a given interpreter on Zepp
 
 <img src="/assets/themes/zeppelin/img/screenshots/interpreter_setting.png">
 
+Properties are exported as environment variable when property name is consisted of upper characters, numbers and underscore ([A-Z_0-9]). Otherwise set properties as JVM property.
+
 Each notebook can be binded to multiple Interpreter Settings using setting icon on upper right corner of the notebook.
 
 <img src="/assets/themes/zeppelin/img/screenshots/interpreter_binding.png" width="800px">

lens/pom.xml

@@ -34,7 +34,7 @@
   <url>http://www.apache.org</url>
   
   <properties>
-    <lens.version>2.2.0-beta-incubating</lens.version>
+    <lens.version>2.5.0-beta</lens.version>
     <spring-shell.version>1.1.0.RELEASE</spring-shell.version>
     <hadoop-common.version>2.4.0</hadoop-common.version>
   </properties>

scripts/vagrant/zeppelin-dev/README.md

@@ -13,11 +13,13 @@ limitations under the License.
 -->
 ## Vagrant Virtual Machine for Apache Zeppelin
   
-This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin.  It can also be used to run an existing Zeppelin build if you don't plan to build from source.  For pyspark users, this script also includes several helpful [Python Libraries and one obscure configuration to help with matplotlib plotting inside Zeppelin](#pythonextras)
+This script creates a virtual machine that launches a repeatable, known set of core dependencies required for developing Zeppelin.  It can also be used to run an existing Zeppelin build if you don't plan to build from source.
+For PySpark users, this script includes several helpful [Python Libraries](#python-extras).
+For SparkR users, this script includes several helpful [R Libraries](#r-extras).
  
 ####Installing the required components to launch a virtual machine.
 
-This script requires three applications, [Ansible](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip "Ansible"), [Vagrant](http://www.vagrantup.com/downloads "Vagrant") and [Virtual Box](https://www.virtualbox.org/ "Virtual Box").  All of these applications are freely available as Open Source projects and extremely easy to set up on most operating systems.
+This script requires three applications, [Ansible](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip "Ansible"), [Vagrant](http://www.vagrantup.com "Vagrant") and [Virtual Box](https://www.virtualbox.org/ "Virtual Box").  All of these applications are freely available as Open Source projects and extremely easy to set up on most operating systems.
 
 ### Create a Zeppelin Ready VM in 4 Steps (5 on Windows)
 
@@ -77,23 +79,24 @@ The virtual machine consists of:
  - libfontconfig to avoid phatomJs missing dependency issues
  - openjdk-7-jdk
  - Python addons: pip, matplotlib, scipy, numpy, pandas
+ - [R](https://www.r-project.org/) and R Packages required to run the R Interpreter and the related R tutorial notebook, including:  Knitr, devtools, repr, rCharts, ggplot2, googleVis, mplot, htmltools, base64enc, data.table
  
 ### How to build & run Zeppelin
 
-This assumes you've already cloned the project either on the host machine in the zeppelin-dev directory (to be shared with the guest machine) or cloned directly into a directory while running inside the guest machine.
+This assumes you've already cloned the project either on the host machine in the zeppelin-dev directory (to be shared with the guest machine) or cloned directly into a directory while running inside the guest machine.  The following build steps will also include Python and R support via PySpark and SparkR:
 
 ```
 cd /incubator-zeppelin
-mvn clean package -Pspark-1.5 -Ppyspark -Dhadoop.version=2.2.0 -Phadoop-2.2 -DskipTests
+mvn clean package -Pspark-1.6 -Ppyspark -Phadoop-2.4 -Psparkr -DskipTests
 ./bin/zeppelin-daemon.sh start
 ```
 
 On your host machine browse to `http://localhost:8080/`
 
-If you [turned off port forwarding](#tweakvm) in the `Vagrantfile` browse to `http://192.168.51.52:8080`
+If you [turned off port forwarding](#tweaking-the-virtual-machine) in the `Vagrantfile` browse to `http://192.168.51.52:8080`
 
 
-### [Tweaking the Virtual Machine](id:tweakvm)
+### Tweaking the Virtual Machine
 
 If you plan to run this virtual machine along side other Vagrant images, you may wish to bind the virtual machine to a specific IP address, and not use port fowarding from your local host.
 
@@ -108,7 +111,7 @@ config.vm.network "private_network", ip: "192.168.51.52"
 This approach usually is typically required if running other virtual machines that discover each other directly by IP address, such as Spark Masters and Slaves as well as Cassandra Nodes, Elasticsearch Nodes, and other Spark data sources.  You may wish to launch nodes in virtual machines with IP Addresses in a subnet that works for your local network, such as: 192.168.51.53, 192.168.51.54, 192.168.51.53, etc..
 
 
-### [Python Extras](id:pythonextras)
+### Python Extras
 
 With zeppelin running, Numpy, SciPy, Pandas and Matplotlib will be available.  Create a pyspark notebook, and try
 
@@ -160,8 +163,7 @@ plt.title('How fast do you want to go today?')
 show(plt)
 ``` 
 
+### R Extras
 
-
-
-
-
+With zeppelin running, an R Tutorial notebook will be available.  The R packages required to run the examples and graphs in this tutorial notebook were installed by this virtual machine.
+The installed R Packages include: Knitr, devtools, repr, rCharts, ggplot2, googleVis, mplot, htmltools, base64enc, data.table

spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -91,7 +91,12 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
   public PySparkInterpreter(Properties property) {
     super(property);
 
-    scriptPath = System.getProperty("java.io.tmpdir") + "/zeppelin_pyspark.py";
+    try {
+      File scriptFile = File.createTempFile("zeppelin_pyspark-", ".py");
+      scriptPath = scriptFile.getAbsolutePath();
+    } catch (IOException e) {
+      throw new InterpreterException(e);
+    }
   }
 
   private void createPythonScript() {
@@ -235,6 +240,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
   @Override
   public void close() {
     executor.getWatchdog().destroyProcess();
+    new File(scriptPath).delete();
     gatewayServer.shutdown();
   }
 

spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -254,7 +254,7 @@ public class SparkInterpreter extends Interpreter {
   }
 
   public SparkContext createSparkContext() {
-    System.err.println("------ Create new SparkContext " + getProperty("master") + " -------");
+    logger.info("------ Create new SparkContext {} -------", getProperty("master"));
 
     String execUri = System.getenv("SPARK_EXECUTOR_URI");
     String[] jars = SparkILoop.getAddedJars();

spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java

@@ -111,8 +111,12 @@ public class ZeppelinR implements ExecuteResultHandler {
     this.rCmdPath = rCmdPath;
     this.libPath = libPath;
     this.port = sparkRBackendPort;
-    scriptPath = System.getProperty("java.io.tmpdir") + "/zeppelin_sparkr.R";
-
+    try {
+      File scriptFile = File.createTempFile("zeppelin_sparkr-", ".R");
+      scriptPath = scriptFile.getAbsolutePath();
+    } catch (IOException e) {
+      throw new InterpreterException(e);
+    }
   }
 
   /**

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java

@@ -71,7 +71,7 @@ public class RemoteInterpreter extends Interpreter {
     this.interpreterRunner = interpreterRunner;
     this.interpreterPath = interpreterPath;
     this.localRepoPath = localRepoPath;
-    env = new HashMap<String, String>();
+    env = getEnvFromInterpreterProperty(property);
     this.connectTimeout = connectTimeout;
     this.maxPoolSize = maxPoolSize;
     this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
@@ -92,12 +92,31 @@ public class RemoteInterpreter extends Interpreter {
     this.interpreterRunner = interpreterRunner;
     this.interpreterPath = interpreterPath;
     this.localRepoPath = localRepoPath;
+    env.putAll(getEnvFromInterpreterProperty(property));
     this.env = env;
     this.connectTimeout = connectTimeout;
     this.maxPoolSize = 10;
     this.remoteInterpreterProcessListener = remoteInterpreterProcessListener;
   }
 
+  private Map<String, String> getEnvFromInterpreterProperty(Properties property) {
+    Map<String, String> env = new HashMap<String, String>();
+    for (Object key : property.keySet()) {
+      if (isEnvString((String) key)) {
+        env.put((String) key, property.getProperty((String) key));
+      }
+    }
+    return env;
+  }
+
+  static boolean isEnvString(String key) {
+    if (key == null || key.length() == 0) {
+      return false;
+    }
+
+    return key.matches("^[A-Z_0-9]*");
+  }
+
   @Override
   public String getClassName() {
     return className;

zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java

@@ -151,6 +151,7 @@ public class RemoteInterpreterServer
       Class<Interpreter> replClass = (Class<Interpreter>) Object.class.forName(className);
       Properties p = new Properties();
       p.putAll(properties);
+      setSystemProperty(p);
 
       Constructor<Interpreter> constructor =
           replClass.getConstructor(new Class[] {Properties.class});
@@ -179,6 +180,19 @@ public class RemoteInterpreterServer
     }
   }
 
+  private void setSystemProperty(Properties properties) {
+    for (Object key : properties.keySet()) {
+      if (!RemoteInterpreter.isEnvString((String) key)) {
+        String value = properties.getProperty((String) key);
+        if (value == null || value.isEmpty()) {
+          System.clearProperty((String) key);
+        } else {
+          System.setProperty((String) key, properties.getProperty((String) key));
+        }
+      }
+    }
+  }
+
   private Interpreter getInterpreter(String noteId, String className) throws TException {
     if (interpreterGroup == null) {
       throw new TException(

zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java

@@ -103,11 +103,16 @@ public abstract class Job {
     this(jobName, listener, JobProgressPoller.DEFAULT_INTERVAL_MSEC);
   }
 
+  public Job(String jobId, String jobName, JobListener listener) {
+    this(jobId, jobName, listener, JobProgressPoller.DEFAULT_INTERVAL_MSEC);
+  }
+
   public Job(String jobId, String jobName, JobListener listener, long progressUpdateIntervalMs) {
     this.jobName = jobName;
     this.listener = listener;
     this.progressUpdateIntervalMs = progressUpdateIntervalMs;
 
+    dateCreated = new Date();
     id = jobId;
 
     setStatus(Status.READY);

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java

@@ -30,6 +30,7 @@ import java.util.Properties;
 import org.apache.thrift.transport.TTransportException;
 import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterEnv;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
 import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client;
 import org.apache.zeppelin.user.AuthenticationInfo;
@@ -701,4 +702,59 @@ public class RemoteInterpreterTest {
     Mockito.verify(client).angularRegistryPush(expected);
   }
 
+  @Test
+  public void testEnvStringPattern() {
+    assertFalse(RemoteInterpreter.isEnvString(null));
+    assertFalse(RemoteInterpreter.isEnvString(""));
+    assertFalse(RemoteInterpreter.isEnvString("abcDEF"));
+    assertFalse(RemoteInterpreter.isEnvString("ABC-DEF"));
+    assertTrue(RemoteInterpreter.isEnvString("ABCDEF"));
+    assertTrue(RemoteInterpreter.isEnvString("ABC_DEF"));
+    assertTrue(RemoteInterpreter.isEnvString("ABC_DEF123"));
+  }
+
+  @Test
+  public void testEnvronmentAndPropertySet() {
+    Properties p = new Properties();
+    p.setProperty("MY_ENV1", "env value 1");
+    p.setProperty("my.property.1", "property value 1");
+
+    RemoteInterpreter intp = new RemoteInterpreter(
+        p,
+        "note",
+        MockInterpreterEnv.class.getName(),
+        new File("../bin/interpreter.sh").getAbsolutePath(),
+        "fake",
+        "fakeRepo",
+        env,
+        10 * 1000,
+        null);
+
+    intpGroup.put("note", new LinkedList<Interpreter>());
+    intpGroup.get("note").add(intp);
+    intp.setInterpreterGroup(intpGroup);
+
+    intp.open();
+
+    InterpreterContext context = new InterpreterContext(
+        "noteId",
+        "id",
+        "title",
+        "text",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        new AngularObjectRegistry(intpGroup.getId(), null),
+        new LocalResourcePool("pool1"),
+        new LinkedList<InterpreterContextRunner>(), null);
+
+
+    assertEquals("env value 1", intp.interpret("getEnv MY_ENV1", context).message());
+    assertEquals("", intp.interpret("getProperty MY_ENV1", context).message());
+    assertEquals("", intp.interpret("getEnv my.property.1", context).message());
+    assertEquals("property value 1", intp.interpret("getProperty my.property.1", context).message());
+
+    intp.close();
+  }
+
 }

zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/mock/MockInterpreterEnv.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.interpreter.remote.mock;
+
+import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.scheduler.Scheduler;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+
+import java.util.List;
+import java.util.Properties;
+
+
+public class MockInterpreterEnv extends Interpreter {
+  static {
+    Interpreter.register(
+        "interpreterA",
+        "group1",
+        MockInterpreterA.class.getName(),
+        new InterpreterPropertyBuilder().build());
+
+  }
+
+
+  public MockInterpreterEnv(Properties property) {
+    super(property);
+  }
+
+  @Override
+  public void open() {
+  }
+
+  @Override
+  public void close() {
+  }
+
+  @Override
+  public InterpreterResult interpret(String st, InterpreterContext context) {
+    String[] cmd = st.split(" ");
+    if (cmd[0].equals("getEnv")) {
+      return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getenv(cmd[1]));
+    } else if (cmd[0].equals("getProperty")){
+      return new InterpreterResult(InterpreterResult.Code.SUCCESS, System.getProperty(cmd[1]));
+    } else {
+      return new InterpreterResult(InterpreterResult.Code.ERROR, cmd[0]);
+    }
+  }
+
+  @Override
+  public void cancel(InterpreterContext context) {
+
+  }
+
+  @Override
+  public FormType getFormType() {
+    return FormType.NATIVE;
+  }
+
+  @Override
+  public int getProgress(InterpreterContext context) {
+    return 0;
+  }
+
+  @Override
+  public List<String> completion(String buf, int cursor) {
+    return null;
+  }
+
+  @Override
+  public Scheduler getScheduler() {
+    return SchedulerFactory.singleton().createOrGetFIFOScheduler("interpreter_" + this.hashCode());
+  }
+}
+

zeppelin-web/bower.json

@@ -44,6 +44,7 @@
       "main": [
         "src-noconflict/ace.js",
         "src-noconflict/mode-scala.js",
+        "src-noconflict/mode-python.js",
         "src-noconflict/mode-sql.js",
         "src-noconflict/mode-markdown.js",
         "src-noconflict/mode-sh.js",

zeppelin-web/src/app/home/home.html

@@ -29,7 +29,8 @@ limitations under the License.
           <h4>Notebook
             <i ng-class="isReloadingNotes ? 'fa fa-refresh fa-spin' : 'fa fa-refresh'"
               ng-style="!isReloadingNotes && {'cursor': 'pointer'}" style="font-size: 13px;"
-              ng-click="reloadNotebookList();">
+              ng-click="reloadNotebookList();"
+              tooltip-placement="bottom" tooltip="Reload notes from storage">
             </i>
           </h4>
 

zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js

@@ -79,7 +79,8 @@ angular.module('zeppelinWebApp')
   var angularObjectRegistry = {};
 
   var editorModes = {
-    'ace/mode/scala': /^%spark/,
+    'ace/mode/python': /^%(\w*\.)?pyspark\s*$/,
+    'ace/mode/scala': /^%(\w*\.)?spark\s*$/,
     'ace/mode/sql': /^%(\w*\.)?\wql/,
     'ace/mode/markdown': /^%md/,
     'ace/mode/sh': /^%sh/
@@ -167,7 +168,7 @@ angular.module('zeppelinWebApp')
         angular.element('#p' + $scope.paragraph.id + '_text').bind('mousewheel', function(e) {
           $scope.keepScrollDown = false;
         });
-
+        $scope.flushStreamingOutput = true;
       } else {
         $timeout(retryRenderer, 10);
       }
@@ -445,13 +446,17 @@ angular.module('zeppelinWebApp')
 
   $scope.$on('appendParagraphOutput', function(event, data) {
     if ($scope.paragraph.id === data.paragraphId) {
+      if ($scope.flushStreamingOutput) {
+        $scope.clearTextOutput();
+        $scope.flushStreamingOutput = false;
+      }
       $scope.appendTextOutput(data.data);
     }
   });
 
   $scope.$on('updateParagraphOutput', function(event, data) {
     if ($scope.paragraph.id === data.paragraphId) {
-      $scope.clearTextOutput(data.data);
+      $scope.clearTextOutput();
       $scope.appendTextOutput(data.data);
     }
   });

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java

@@ -167,7 +167,9 @@ public class Note implements Serializable, JobListener {
    * @param srcParagraph
    */
   public void addCloneParagraph(Paragraph srcParagraph) {
-    Paragraph newParagraph = new Paragraph(this, this, replLoader);
+
+    // Keep paragraph original ID
+    final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, replLoader);
 
     Map<String, Object> config = new HashMap<>(srcParagraph.getConfig());
     Map<String, Object> param = new HashMap<>(srcParagraph.settings.getParams());

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java

@@ -17,6 +17,7 @@
 
 package org.apache.zeppelin.notebook;
 
+import org.apache.zeppelin.display.AngularObject;
 import org.apache.zeppelin.display.AngularObjectRegistry;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.apache.zeppelin.display.GUI;
@@ -61,6 +62,19 @@ public class Paragraph extends Job implements Serializable, Cloneable {
     settings = new GUI();
   }
 
+  public Paragraph(String paragraphId, Note note, JobListener listener,
+                   NoteInterpreterLoader replLoader) {
+    super(paragraphId, generateId(), listener);
+    this.note = note;
+    this.replLoader = replLoader;
+    title = null;
+    text = null;
+    authenticationInfo = null;
+    dateUpdated = null;
+    settings = new GUI();
+    config = new HashMap<String, Object>();
+  }
+
   public Paragraph(Note note, JobListener listener, NoteInterpreterLoader replLoader) {
     super(generateId(), listener);
     this.note = note;
@@ -232,6 +246,12 @@ public class Paragraph extends Job implements Serializable, Cloneable {
       String scriptBody = getScriptBody();
       Map<String, Input> inputs = Input.extractSimpleQueryParam(scriptBody); // inputs will be built
                                                                              // from script body
+
+      final AngularObjectRegistry angularRegistry = repl.getInterpreterGroup()
+              .getAngularObjectRegistry();
+
+      scriptBody = extractVariablesFromAngularRegistry(scriptBody, inputs, angularRegistry);
+
       settings.setForms(inputs);
       script = Input.getSimpleQuery(settings.getParams(), scriptBody);
     }
@@ -390,4 +410,25 @@ public class Paragraph extends Job implements Serializable, Cloneable {
     Paragraph paraClone = (Paragraph) this.clone();
     return paraClone;
   }
+
+  String extractVariablesFromAngularRegistry(String scriptBody, Map<String, Input> inputs,
+                                             AngularObjectRegistry angularRegistry) {
+
+    final String noteId = this.getNote().getId();
+    final String paragraphId = this.getId();
+
+    final Set<String> keys = new HashSet<>(inputs.keySet());
+
+    for (String varName : keys) {
+      final AngularObject paragraphScoped = angularRegistry.get(varName, noteId, paragraphId);
+      final AngularObject noteScoped = angularRegistry.get(varName, noteId, null);
+      final AngularObject angularObject = paragraphScoped != null ? paragraphScoped : noteScoped;
+      if (angularObject != null) {
+        inputs.remove(varName);
+        final String pattern = "[$][{]\\s*" + varName + "\\s*(?:=[^}]+)?[}]";
+        scriptBody = scriptBody.replaceAll(pattern, angularObject.get().toString());
+      }
+    }
+    return scriptBody;
+  }
 }

zeppelin-zengine/src/test/java/org/apache/zeppelin/display/AngularObjectBuilder.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.display;
+
+public class AngularObjectBuilder {
+
+    public static <T> AngularObject<T> build(String varName, T value, String noteId,
+                                             String paragraphId) {
+        return new AngularObject<>(varName, value, noteId, paragraphId, null);
+    }
+}

zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java

@@ -322,7 +322,9 @@ public class NotebookTest implements JobListenerFactory{
     Note cloneNote = notebook.cloneNote(note.getId(), "clone note");
     Paragraph cp = cloneNote.paragraphs.get(0);
     assertEquals(cp.getStatus(), Status.READY);
-    assertNotEquals(cp.getId(), p.getId());
+
+    // Keep same ParagraphID
+    assertEquals(cp.getId(), p.getId());
     assertEquals(cp.text, p.text);
     assertEquals(cp.getResult().message(), p.getResult().message());
   }

zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java

@@ -17,9 +17,20 @@
 
 package org.apache.zeppelin.notebook;
 
-import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.zeppelin.display.AngularObject;
+import org.apache.zeppelin.display.AngularObjectBuilder;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.Input;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
 
 public class ParagraphTest {
   @Test
@@ -35,4 +46,43 @@ public class ParagraphTest {
     String text = "12345678";
     assertEquals(text, Paragraph.getScriptBody(text));
   }
+
+  @Test
+  public void should_extract_variable_from_angular_object_registry() throws Exception {
+    //Given
+    final String noteId = "noteId";
+
+    final AngularObjectRegistry registry = mock(AngularObjectRegistry.class);
+    final Note note = mock(Note.class);
+    final Map<String, Input> inputs = new HashMap<>();
+    inputs.put("name", null);
+    inputs.put("age", null);
+    inputs.put("job", null);
+
+    final String scriptBody = "My name is ${name} and I am ${age=20} years old. " +
+            "My occupation is ${ job = engineer | developer | artists}";
+
+    final Paragraph paragraph = new Paragraph(note, null, null);
+    final String paragraphId = paragraph.getId();
+
+    final AngularObject nameAO = AngularObjectBuilder.build("name", "DuyHai DOAN", noteId,
+            paragraphId);
+
+    final AngularObject ageAO = AngularObjectBuilder.build("age", 34, noteId, null);
+
+    when(note.getId()).thenReturn(noteId);
+    when(registry.get("name", noteId, paragraphId)).thenReturn(nameAO);
+    when(registry.get("age", noteId, null)).thenReturn(ageAO);
+
+    final String expected = "My name is DuyHai DOAN and I am 34 years old. " +
+            "My occupation is ${ job = engineer | developer | artists}";
+    //When
+    final String actual = paragraph.extractVariablesFromAngularRegistry(scriptBody, inputs,
+            registry);
+
+    //Then
+    verify(registry).get("name", noteId, paragraphId);
+    verify(registry).get("age", noteId, null);
+    assertEquals(actual, expected);
+  }
 }