Merge branch 'master' of https://github.com/apache/zeppelin into ZEPPELIN-1564

.travis.yml

@@ -36,7 +36,7 @@ matrix:
   include:
     # Test License compliance using RAT tool
     - jdk: "oraclejdk7"
-      env: SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.3" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
+      env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
 
     # Test all modules with spark 2.0.0 and scala 2.11
     - jdk: "oraclejdk7"
@@ -58,18 +58,6 @@ matrix:
     - jdk: "oraclejdk7"
       env: SCALA_VER="2.10" SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Pr -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,r -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
 
-    # Test spark module for 1.3.1
-    - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
-
-    # Test spark module for 1.2.2
-    - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.2.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
-
-    # Test spark module for 1.1.1
-    - jdk: "oraclejdk7"
-      env: SCALA_VER="2.10" SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
-
     # Test selenium with spark module for 1.6.1
     - jdk: "oraclejdk7"
       env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"

README.md

@@ -128,9 +128,6 @@ Available profiles are
 -Pspark-1.6
 -Pspark-1.5
 -Pspark-1.4
--Pspark-1.3
--Pspark-1.2
--Pspark-1.1
 -Pcassandra-spark-1.5
 -Pcassandra-spark-1.4
 -Pcassandra-spark-1.3
@@ -192,7 +189,7 @@ enable 3rd party vendor repository (cloudera)
 
 ##### `-Pmapr[version]` (optional)
 
-For the MapR Hadoop Distribution, these profiles will handle the Hadoop version. As MapR allows different versions of Spark to be installed, you should specify which version of Spark is installed on the cluster by adding a Spark profile (`-Pspark-1.2`, `-Pspark-1.3`, etc.) as needed.
+For the MapR Hadoop Distribution, these profiles will handle the Hadoop version. As MapR allows different versions of Spark to be installed, you should specify which version of Spark is installed on the cluster by adding a Spark profile (`-Pspark-1.6`, `-Pspark-2.0`, etc.) as needed.
 The correct Maven artifacts can be found for every version of MapR at http://doc.mapr.com
 
 Available profiles are

conf/shiro.ini.template

@@ -78,5 +78,5 @@ admin = *
 #/api/interpreter/** = authc, roles[admin]
 #/api/configurations/** = authc, roles[admin]
 #/api/credential/** = authc, roles[admin]
-/** = anon
-#/** = authc
+#/** = anon
+/** = authc

docs/install/upgrade.md

@@ -52,3 +52,4 @@ So, copying `notebook` and `conf` directory should be enough.
  - From 0.7, we don't use `ZEPPELIN_JAVA_OPTS` as default value of `ZEPPELIN_INTP_JAVA_OPTS` and also the same for `ZEPPELIN_MEM`/`ZEPPELIN_INTP_MEM`. If user want to configure the jvm opts of interpreter process, please set `ZEPPELIN_INTP_JAVA_OPTS` and `ZEPPELIN_INTP_MEM` explicitly. If you don't set `ZEPPELIN_INTP_MEM`, Zeppelin will set it to `-Xms1024m -Xmx1024m -XX:MaxPermSize=512m` by default.
  - Mapping from `%jdbc(prefix)` to `%prefix` is no longer available. Instead, you can use %[interpreter alias] with multiple interpreter setttings on GUI.
  - Usage of `ZEPPELIN_PORT` is not supported in ssl mode. Instead use `ZEPPELIN_SSL_PORT` to configure the ssl port. Value from `ZEPPELIN_PORT` is used only when `ZEPPELIN_SSL` is set to `false`.
+ - The support on Spark 1.1.x to 1.3.x is deprecated.

docs/security/authentication.md

@@ -85,7 +85,7 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
         }
 
         location /ws {  # For websocket support
-            proxy_pass http://zeppelin;
+            proxy_pass http://zeppelin/ws;
             proxy_http_version 1.1;
             proxy_set_header Upgrade websocket;
             proxy_set_header Connection upgrade;
@@ -130,4 +130,4 @@ This instruction based on Ubuntu 14.04 LTS but may work with other OS with few c
 Another option is to have an authentication server that can verify user credentials in an LDAP server.
 If an incoming request to the Zeppelin server does not have a cookie with user information encrypted with the authentication server public key, the user
 is redirected to the authentication server. Once the user is verified, the authentication server redirects the browser to a specific URL in the Zeppelin server which sets the authentication cookie in the browser.
-The end result is that all requests to the Zeppelin web server have the authentication cookie which contains user and groups information.
+The end result is that all requests to the Zeppelin web server have the authentication cookie which contains user and groups information.

spark/pom.xml

@@ -450,38 +450,6 @@
   </build>
 
   <profiles>
-    <profile>
-      <id>spark-1.1</id>
-      <dependencies>
-
-      </dependencies>
-      <properties>
-        <spark.version>1.1.1</spark.version>
-        <akka.version>2.2.3-shaded-protobuf</akka.version>
-      </properties>
-    </profile>
-
-    <profile>
-      <id>spark-1.2</id>
-      <dependencies>
-      </dependencies>
-      <properties>
-        <spark.version>1.2.1</spark.version>
-      </properties>
-    </profile>
-
-    <profile>
-      <id>spark-1.3</id>
-
-      <properties>
-        <spark.version>1.3.1</spark.version>
-      </properties>
-
-      <dependencies>
-      </dependencies>
-
-    </profile>
-
     <profile>
       <id>spark-1.4</id>
       <properties>

spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java

@@ -49,9 +49,9 @@ import org.apache.zeppelin.interpreter.Interpreter;
 import org.apache.zeppelin.interpreter.InterpreterContext;
 import org.apache.zeppelin.interpreter.InterpreterException;
 import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
 import org.apache.zeppelin.interpreter.WrappedInterpreter;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
 import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
 import org.apache.zeppelin.spark.dep.SparkDependencyContext;
 import org.slf4j.Logger;
@@ -165,6 +165,15 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
     }
   }
 
+  private Map setupPySparkEnv() throws IOException{
+    Map env = EnvironmentUtils.getProcEnvironment();
+    if (!env.containsKey("PYTHONPATH")) {
+      SparkConf conf = getSparkConf();
+      env.put("PYTHONPATH", conf.get("spark.submit.pyFiles").replaceAll(",", ":"));
+    }
+    return env;
+  }
+
   private void createGatewayServerAndStartScript() {
     // create python script
     createPythonScript();
@@ -196,10 +205,8 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
     executor.setStreamHandler(streamHandler);
     executor.setWatchdog(new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT));
 
-
     try {
-      Map env = EnvironmentUtils.getProcEnvironment();
-
+      Map env = setupPySparkEnv();
       executor.execute(cmd, env, this);
       pythonscriptRunning = true;
     } catch (IOException e) {

spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java

@@ -504,6 +504,7 @@ public class SparkInterpreter extends Interpreter {
         conf.set("spark.files", conf.get("spark.yarn.dist.files"));
       }
       conf.set("spark.submit.pyArchives", Joiner.on(":").join(pythonLibs));
+      conf.set("spark.submit.pyFiles", Joiner.on(",").join(pythonLibUris));
     }
 
     // Distributes needed libraries to workers

spark/src/test/java/org/apache/zeppelin/spark/PySparkInterpreterTest.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.spark;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.GUI;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutputListener;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Properties;
+
+import static org.junit.Assert.*;
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class PySparkInterpreterTest {
+  public static SparkInterpreter sparkInterpreter;
+  public static PySparkInterpreter pySparkInterpreter;
+  public static InterpreterGroup intpGroup;
+  private File tmpDir;
+  public static Logger LOGGER = LoggerFactory.getLogger(PySparkInterpreterTest.class);
+  private InterpreterContext context;
+
+  public static Properties getPySparkTestProperties() {
+    Properties p = new Properties();
+    p.setProperty("master", "local[*]");
+    p.setProperty("spark.app.name", "Zeppelin Test");
+    p.setProperty("zeppelin.spark.useHiveContext", "true");
+    p.setProperty("zeppelin.spark.maxResult", "1000");
+    p.setProperty("zeppelin.spark.importImplicit", "true");
+    p.setProperty("zeppelin.pyspark.python", "python");
+    return p;
+  }
+
+  /**
+   * Get spark version number as a numerical value.
+   * eg. 1.1.x => 11, 1.2.x => 12, 1.3.x => 13 ...
+   */
+  public static int getSparkVersionNumber() {
+    if (sparkInterpreter == null) {
+      return 0;
+    }
+
+    String[] split = sparkInterpreter.getSparkContext().version().split("\\.");
+    int version = Integer.parseInt(split[0]) * 10 + Integer.parseInt(split[1]);
+    return version;
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
+    System.setProperty("zeppelin.dep.localrepo", tmpDir.getAbsolutePath() + "/local-repo");
+    tmpDir.mkdirs();
+
+    intpGroup = new InterpreterGroup();
+    intpGroup.put("note", new LinkedList<Interpreter>());
+
+    if (sparkInterpreter == null) {
+      sparkInterpreter = new SparkInterpreter(getPySparkTestProperties());
+      intpGroup.get("note").add(sparkInterpreter);
+      sparkInterpreter.setInterpreterGroup(intpGroup);
+      sparkInterpreter.open();
+    }
+
+    if (pySparkInterpreter == null) {
+      pySparkInterpreter = new PySparkInterpreter(getPySparkTestProperties());
+      intpGroup.get("note").add(pySparkInterpreter);
+      pySparkInterpreter.setInterpreterGroup(intpGroup);
+      pySparkInterpreter.open();
+    }
+
+    context = new InterpreterContext("note", "id", "title", "text",
+      new AuthenticationInfo(),
+      new HashMap<String, Object>(),
+      new GUI(),
+      new AngularObjectRegistry(intpGroup.getId(), null),
+      new LocalResourcePool("id"),
+      new LinkedList<InterpreterContextRunner>(),
+      new InterpreterOutput(new InterpreterOutputListener() {
+        @Override
+        public void onAppend(InterpreterOutput out, byte[] line) {
+
+        }
+
+        @Override
+        public void onUpdate(InterpreterOutput out, byte[] output) {
+
+        }
+      }));
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    delete(tmpDir);
+  }
+
+  private void delete(File file) {
+    if (file.isFile()) file.delete();
+    else if (file.isDirectory()) {
+      File[] files = file.listFiles();
+      if (files != null && files.length > 0) {
+        for (File f : files) {
+          delete(f);
+        }
+      }
+      file.delete();
+    }
+  }
+
+  @Test
+  public void testBasicIntp() {
+    if (getSparkVersionNumber() > 11) {
+      assertEquals(InterpreterResult.Code.SUCCESS,
+        pySparkInterpreter.interpret("a = 1\n", context).code());
+    }
+  }
+
+}

testing/downloadSpark.sh

@@ -16,28 +16,15 @@
 # limitations under the License.
 #
 
-
 if [[ "$#" -ne 2 ]]; then
     echo "usage) $0 [spark version] [hadoop version]"
     echo "   eg) $0 1.3.1 2.6"
-    exit 1
+    exit 0
 fi
 
 SPARK_VERSION="${1}"
 HADOOP_VERSION="${2}"
 
-echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
-if [[ "$?" -eq 0 ]]; then
-  echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
-  if [[ "$?" -eq 0 ]]; then
-    SPARK_VER_RANGE="<=1.2"
-  else
-    SPARK_VER_RANGE="<=1.3"
-  fi
-else
-  SPARK_VER_RANGE=">1.3"
-fi
-
 set -xe
 
 MAX_DOWNLOAD_TIME_SEC=590
@@ -75,30 +62,13 @@ if [[ ! -d "${SPARK_HOME}" ]]; then
         ls -la .
         echo "${SPARK_CACHE} does not have ${SPARK_ARCHIVE} downloading ..."
 
-        # download archive if not cached
-        if [[ "${SPARK_VERSION}" = "1.1.1" || "${SPARK_VERSION}" = "1.2.2" || "${SPARK_VERSION}" = "1.3.1" || "${SPARK_VERSION}" = "1.4.1" ]]; then
-            echo "${SPARK_VERSION} being downloaded from archives"
-            # spark old versions are only available only on the archives (prior to 1.5.2)
-            STARTTIME=`date +%s`
-            #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
-            download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
-            ENDTIME=`date +%s`
-            DOWNLOADTIME="$((ENDTIME-STARTTIME))"
-        else
-            echo "${SPARK_VERSION} being downloaded from mirror"
-            # spark 1.5.2 and up and later can be downloaded from mirror
-            # get download address from mirror
-            MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz?asjson=1")
-
-            PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
-            PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
-
-            STARTTIME=`date +%s`
-            #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget -q "${PREFFERED}${PATHINFO}"
-            download_with_retry "${PREFFERED}${PATHINFO}"
-            ENDTIME=`date +%s`
-            DOWNLOADTIME="$((ENDTIME-STARTTIME))"
-        fi
+        # download spark from archive if not cached
+        echo "${SPARK_VERSION} being downloaded from archives"
+        STARTTIME=`date +%s`
+        #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
+        download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
+        ENDTIME=`date +%s`
+        DOWNLOADTIME="$((ENDTIME-STARTTIME))"
     fi
 
     # extract archive in un-cached root, clean-up on failure

zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java

@@ -26,7 +26,10 @@ import java.util.Set;
 import javax.servlet.DispatcherType;
 import javax.ws.rs.core.Application;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.cxf.jaxrs.servlet.CXFNonSpringJaxrsServlet;
+import org.apache.shiro.web.env.EnvironmentLoaderListener;
+import org.apache.shiro.web.servlet.ShiroFilter;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
 import org.apache.zeppelin.dep.DependencyResolver;
@@ -259,15 +262,16 @@ public class ZeppelinServer extends Application {
     webapp.setSessionHandler(new SessionHandler());
     webapp.addServlet(cxfServletHolder, "/api/*");
 
-    webapp.setInitParameter("shiroConfigLocations",
-        new File(conf.getShiroPath()).toURI().toString());
-
-    SecurityUtils.initSecurityManager(conf.getShiroPath());
-    webapp.addFilter(org.apache.shiro.web.servlet.ShiroFilter.class, "/api/*",
-        EnumSet.allOf(DispatcherType.class));
-
-    webapp.addEventListener(new org.apache.shiro.web.env.EnvironmentLoaderListener());
-
+    String shiroIniPath = conf.getShiroPath();
+    if (!StringUtils.isBlank(shiroIniPath)) {
+      webapp.setInitParameter("shiroConfigLocations", new File(shiroIniPath).toURI().toString());
+      SecurityUtils.initSecurityManager(shiroIniPath);
+      webapp.addFilter(ShiroFilter.class, "/api/*", EnumSet.allOf(DispatcherType.class));
+      webapp.addEventListener(new EnvironmentLoaderListener());
+    } else {
+      webapp.addFilter(new FilterHolder(CorsFilter.class),
+          "/api/*", EnumSet.allOf(DispatcherType.class));
+    }
   }
 
   private static WebAppContext setupWebAppContext(ContextHandlerCollection contexts,

zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java

@@ -715,39 +715,49 @@ public class NotebookServer extends WebSocketServlet implements
 
     return cronUpdated;
   }
+
   private void createNote(NotebookSocket conn, HashSet<String> userAndRoles,
                           Notebook notebook, Message message)
       throws IOException {
     AuthenticationInfo subject = new AuthenticationInfo(message.principal);
-    Note note = null;
 
-    String defaultInterpreterId = (String) message.get("defaultInterpreterId");
-    if (!StringUtils.isEmpty(defaultInterpreterId)) {
-      List<String> interpreterSettingIds = new LinkedList<>();
-      interpreterSettingIds.add(defaultInterpreterId);
-      for (String interpreterSettingId : notebook.getInterpreterFactory().
-              getDefaultInterpreterSettingList()) {
-        if (!interpreterSettingId.equals(defaultInterpreterId)) {
-          interpreterSettingIds.add(interpreterSettingId);
+    try {
+      Note note = null;
+
+      String defaultInterpreterId = (String) message.get("defaultInterpreterId");
+      if (!StringUtils.isEmpty(defaultInterpreterId)) {
+        List<String> interpreterSettingIds = new LinkedList<>();
+        interpreterSettingIds.add(defaultInterpreterId);
+        for (String interpreterSettingId : notebook.getInterpreterFactory().
+                getDefaultInterpreterSettingList()) {
+          if (!interpreterSettingId.equals(defaultInterpreterId)) {
+            interpreterSettingIds.add(interpreterSettingId);
+          }
         }
+        note = notebook.createNote(interpreterSettingIds, subject);
+      } else {
+        note = notebook.createNote(subject);
       }
-      note = notebook.createNote(interpreterSettingIds, subject);
-    } else {
-      note = notebook.createNote(subject);
-    }
 
-    note.addParagraph(); // it's an empty note. so add one paragraph
-    if (message != null) {
-      String noteName = (String) message.get("name");
-      if (StringUtils.isEmpty(noteName)){
-        noteName = "Note " + note.getId();
+      note.addParagraph(); // it's an empty note. so add one paragraph
+      if (message != null) {
+        String noteName = (String) message.get("name");
+        if (StringUtils.isEmpty(noteName)) {
+          noteName = "Note " + note.getId();
+        }
+        note.setName(noteName);
       }
-      note.setName(noteName);
-    }
 
-    note.persist(subject);
-    addConnectionToNote(note.getId(), (NotebookSocket) conn);
-    conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
+      note.persist(subject);
+      addConnectionToNote(note.getId(), (NotebookSocket) conn);
+      conn.send(serializeMessage(new Message(OP.NEW_NOTE).put("note", note)));
+    } catch (FileSystemException e) {
+      LOG.error("Exception from createNote", e);
+      conn.send(serializeMessage(new Message(OP.ERROR_INFO).put("info",
+                "Oops! There is something wrong with the notebook file system. "
+                + "Please check the logs for more details.")));
+      return;
+    }
     broadcastNoteList(subject, userAndRoles);
   }
 

zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java

@@ -16,30 +16,41 @@
  */
 package org.apache.zeppelin.utils;
 
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.shiro.config.IniSecurityManagerFactory;
+import org.apache.shiro.mgt.SecurityManager;
 import org.apache.shiro.realm.Realm;
 import org.apache.shiro.realm.text.IniRealm;
 import org.apache.shiro.subject.Subject;
 import org.apache.shiro.util.ThreadContext;
 import org.apache.shiro.web.mgt.DefaultWebSecurityManager;
-import org.apache.shiro.mgt.SecurityManager;
-import org.apache.shiro.config.IniSecurityManagerFactory;
 import org.apache.zeppelin.conf.ZeppelinConfiguration;
 
-import java.net.InetAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
-import java.util.*;
+import com.google.common.collect.Sets;
 
 /**
  * Tools for securing Zeppelin
  */
 public class SecurityUtils {
 
+  private static final String ANONYMOUS = "anonymous";
+  private static final HashSet<String> EMPTY_HASHSET = Sets.newHashSet();
+  private static boolean isEnabled = false;
+  
   public static void initSecurityManager(String shiroPath) {
     IniSecurityManagerFactory factory = new IniSecurityManagerFactory("file:" + shiroPath);
     SecurityManager securityManager = factory.getInstance();
     org.apache.shiro.SecurityUtils.setSecurityManager(securityManager);
+    isEnabled = true;
   }
 
   public static Boolean isValidOrigin(String sourceHost, ZeppelinConfiguration conf)
@@ -65,18 +76,24 @@ public class SecurityUtils {
    * @return shiro principal
    */
   public static String getPrincipal() {
+    if (!isEnabled) {
+      return ANONYMOUS;
+    }
     Subject subject = org.apache.shiro.SecurityUtils.getSubject();
 
     String principal;
     if (subject.isAuthenticated()) {
       principal = subject.getPrincipal().toString();
     } else {
-      principal = "anonymous";
+      principal = ANONYMOUS;
     }
     return principal;
   }
 
   public static Collection getRealmsList() {
+    if (!isEnabled) {
+      return Collections.emptyList();
+    }
     DefaultWebSecurityManager defaultWebSecurityManager;
     String key = ThreadContext.SECURITY_MANAGER_KEY;
     defaultWebSecurityManager = (DefaultWebSecurityManager) ThreadContext.get(key);
@@ -91,6 +108,9 @@ public class SecurityUtils {
    * @return shiro roles
    */
   public static HashSet<String> getRoles() {
+    if (!isEnabled) {
+      return EMPTY_HASHSET;
+    }
     Subject subject = org.apache.shiro.SecurityUtils.getSubject();
     HashSet<String> roles = new HashSet<>();
     Map allRoles = null;
@@ -123,6 +143,9 @@ public class SecurityUtils {
    * Checked if shiro enabled or not
    */
   public static boolean isAuthenticated() {
+    if (!isEnabled) {
+      return false;
+    }
     return org.apache.shiro.SecurityUtils.getSubject().isAuthenticated();
   }
 }

zeppelin-server/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java

@@ -16,6 +16,13 @@
  */
 package org.apache.zeppelin.integration;
 
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.zeppelin.AbstractZeppelinIT;
@@ -34,11 +41,6 @@ import org.openqa.selenium.WebElement;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.util.List;
-
 
 /**
  * Created for org.apache.zeppelin.integration on 13/06/16.
@@ -48,7 +50,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
 
   @Rule
   public ErrorCollector collector = new ErrorCollector();
-
+  static String shiroPath;
   static String authShiro = "[users]\n" +
       "admin = password1, admin\n" +
       "finance1 = finance1, finance\n" +
@@ -80,8 +82,11 @@ public class AuthenticationIT extends AbstractZeppelinIT {
     try {
       System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), "../");
       ZeppelinConfiguration conf = ZeppelinConfiguration.create();
-      File file = new File(conf.getShiroPath());
-      originalShiro = StringUtils.join(FileUtils.readLines(file, "UTF-8"), "\n");
+      shiroPath = conf.getRelativeDir(String.format("%s/shiro.ini", conf.getConfDir()));
+      File file = new File(shiroPath);
+      if (file.exists()) {
+        originalShiro = StringUtils.join(FileUtils.readLines(file, "UTF-8"), "\n");
+      }
       FileUtils.write(file, authShiro, "UTF-8");
     } catch (IOException e) {
       LOG.error("Error in AuthenticationIT startUp::", e);
@@ -97,9 +102,14 @@ public class AuthenticationIT extends AbstractZeppelinIT {
       return;
     }
     try {
-      ZeppelinConfiguration conf = ZeppelinConfiguration.create();
-      File file = new File(conf.getShiroPath());
-      FileUtils.write(file, originalShiro, "UTF-8");
+      if (!StringUtils.isBlank(shiroPath)) {
+        File file = new File(shiroPath);
+        if (StringUtils.isBlank(originalShiro)) {
+          FileUtils.deleteQuietly(file);
+        } else {
+          FileUtils.write(file, originalShiro, "UTF-8");
+        }
+      }
     } catch (IOException e) {
       LOG.error("Error in AuthenticationIT tearDown::", e);
     }
@@ -119,6 +129,23 @@ public class AuthenticationIT extends AbstractZeppelinIT {
     ZeppelinITUtils.sleep(1000, false);
   }
 
+  private void testShowNotebookListOnNavbar() throws Exception {
+    if (!endToEndTestEnabled()) {
+      return;
+    }
+    try {
+      pollingWait(By.xpath("//li[@class='dropdown notebook-list-dropdown']"),
+          MAX_BROWSER_TIMEOUT_SEC).click();
+      assertTrue(driver.findElements(By.xpath("//a[@class=\"notebook-list-item ng-scope\"]")).size() > 0);
+      pollingWait(By.xpath("//li[@class='dropdown notebook-list-dropdown']"),
+              MAX_BROWSER_TIMEOUT_SEC).click();
+      pollingWait(By.xpath("//li[@class='dropdown notebook-list-dropdown']"),
+              MAX_BROWSER_TIMEOUT_SEC).click();
+    } catch (Exception e) {
+      handleException("Exception in ParagraphActionsIT while testShowNotebookListOnNavbar ", e);
+    }
+  }
+
   private void logoutUser(String userName) {
     ZeppelinITUtils.sleep(500, false);
     driver.findElement(By.xpath("//div[contains(@class, 'navbar-collapse')]//li[contains(.,'" +
@@ -144,7 +171,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
 
       authenticationIT.logoutUser("admin");
     } catch (Exception e) {
-      handleException("Exception in ParagraphActionsIT while testCreateNewButton ", e);
+      handleException("Exception in AuthenticationIT while testCreateNewButton ", e);
     }
   }
 

zeppelin-server/src/test/java/org/apache/zeppelin/rest/SecurityRestApiTest.java

@@ -17,8 +17,10 @@
 
 package org.apache.zeppelin.rest;
 
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.hamcrest.CoreMatchers;
 import org.junit.AfterClass;
@@ -27,11 +29,8 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ErrorCollector;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.*;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
 
 public class SecurityRestApiTest extends AbstractTestRestApi {
   Gson gson = new Gson();
@@ -41,7 +40,7 @@ public class SecurityRestApiTest extends AbstractTestRestApi {
 
   @BeforeClass
   public static void init() throws Exception {
-    AbstractTestRestApi.startUp();
+    AbstractTestRestApi.startUpWithAuthenticationEnable();;
   }
 
   @AfterClass

zeppelin-web/src/app/notebook/paragraph/paragraph-parameterizedQueryForm.html

@@ -15,7 +15,7 @@ limitations under the License.
       ng-show="!paragraph.config.tableHide"
       class=" paragraphForm form-horizontal row">
   <div class="form-group col-sm-6 col-md-6 col-lg-4"
-       ng-repeat="formulaire in paragraph.settings.forms"
+       ng-repeat="formulaire in paragraph.settings.forms | toArray | orderBy:'name.toString()'"
        ng-init="loadForm(formulaire, paragraph.settings.params)">
     <label class="control-label input-sm" ng-class="{'disable': paragraph.status == 'RUNNING' || paragraph.status == 'PENDING' }">{{formulaire.name}}</label>
     <div>

zeppelin-web/src/components/navbar/navbar.controller.js

@@ -114,6 +114,7 @@
 
     $scope.$on('setNoteMenu', function(event, notes) {
       noteListDataFactory.setNotes(notes);
+      initNotebookListEventListener();
     });
 
     $scope.$on('setConnectedStatus', function(event, param) {
@@ -127,15 +128,17 @@
     /*
     ** Performance optimization for Browser Render.
     */
-    angular.element(document).ready(function() {
-      angular.element('.notebook-list-dropdown').on('show.bs.dropdown', function() {
-        $scope.isDrawNavbarNoteList = true;
-      });
+    function initNotebookListEventListener() {
+      angular.element(document).ready(function() {
+        angular.element('.notebook-list-dropdown').on('show.bs.dropdown', function() {
+          $scope.isDrawNavbarNoteList = true;
+        });
 
-      angular.element('.notebook-list-dropdown').on('hide.bs.dropdown', function() {
-        $scope.isDrawNavbarNoteList = false;
+        angular.element('.notebook-list-dropdown').on('hide.bs.dropdown', function() {
+          $scope.isDrawNavbarNoteList = false;
+        });
       });
-    });
+    }
   }
 
 })();

zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java

@@ -27,6 +27,7 @@ import java.util.Map;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.XMLConfiguration;
 import org.apache.commons.configuration.tree.ConfigurationNode;
+import org.apache.commons.lang.StringUtils;
 import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
 import org.apache.zeppelin.util.Util;
 import org.slf4j.Logger;
@@ -402,9 +403,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
   }
 
   public String getShiroPath() {
-    String shiroPath =  getRelativeDir(String.format("%s/shiro.ini", getConfDir()));
-    return new File(shiroPath).exists() ? shiroPath
-        : getRelativeDir(String.format("%s/shiro.ini.template", getConfDir()));
+    String shiroPath = getRelativeDir(String.format("%s/shiro.ini", getConfDir()));
+    return new File(shiroPath).exists() ? shiroPath : StringUtils.EMPTY;
   }
 
   public String getInterpreterRemoteRunnerPath() {

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java

@@ -120,7 +120,8 @@ public class Notebook implements NoteEventListener {
     quartzSched.start();
     CronJob.notebook = this;
 
-    loadAllNotes();
+    AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS;
+    loadAllNotes(anonymous);
     if (this.noteSearchService != null) {
       long start = System.nanoTime();
       logger.info("Notebook indexing started...");
@@ -462,11 +463,11 @@ public class Notebook implements NoteEventListener {
     return note;
   }
 
-  private void loadAllNotes() throws IOException {
-    List<NoteInfo> noteInfos = notebookRepo.list(null);
+  void loadAllNotes(AuthenticationInfo subject) throws IOException {
+    List<NoteInfo> noteInfos = notebookRepo.list(subject);
 
     for (NoteInfo info : noteInfos) {
-      loadNoteFromRepo(info.getId(), null);
+      loadNoteFromRepo(info.getId(), subject);
     }
   }
 

zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java

@@ -65,7 +65,7 @@ public class NotebookTest implements JobListenerFactory{
   private DependencyResolver depResolver;
   private NotebookAuthorization notebookAuthorization;
   private Credentials credentials;
-  private AuthenticationInfo anonymous = new AuthenticationInfo("anonymous");
+  private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS;
 
   @Before
   public void setUp() throws Exception {
@@ -196,6 +196,30 @@ public class NotebookTest implements JobListenerFactory{
     assertEquals(notes.size(), 0);
   }
 
+  @Test
+  public void testLoadAllNotes() {
+    Note note;
+    try {
+      assertEquals(0, notebook.getAllNotes().size());
+      note = notebook.createNote(anonymous);
+      Paragraph p1 = note.addParagraph();
+      Map config = p1.getConfig();
+      config.put("enabled", true);
+      p1.setConfig(config);
+      p1.setText("hello world");
+      note.persist(anonymous);
+    } catch (IOException fe) {
+      logger.warn("Failed to create note and paragraph. Possible problem with persisting note, safe to ignore", fe);
+    }
+
+    try {
+      notebook.loadAllNotes(anonymous);
+      assertEquals(1, notebook.getAllNotes().size());
+    } catch (IOException e) {
+      fail("Subject is non-emtpy anonymous, shouldn't fail");
+    }
+  }
+  
   @Test
   public void testPersist() throws IOException, SchedulerException, RepositoryException {
     Note note = notebook.createNote(anonymous);