[ZEPPELIN-3977]. Create shell script for converting old note file to new file

bin/upgrade-note.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Convert note format from 0.9.0 before to 0.9.0 after
+#
+
+USAGE="Usage: bin/upgrade-note.sh [-d]"
+
+bin=$(dirname "${BASH_SOURCE-$0}")
+bin=$(cd "${bin}">/dev/null; pwd)
+
+. "${bin}/common.sh"
+
+JAVA_OPTS="-Dzeppelin.log.file=logs/upgrade-note.log"
+MAIN_CLASS=org.apache.zeppelin.notebook.repo.UpgradeNoteFileTool
+
+# construct classpath
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
+  ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
+fi
+
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
+  ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
+fi
+
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
+  ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
+fi
+
+addJarInDir "${ZEPPELIN_HOME}"
+addJarInDir "${ZEPPELIN_HOME}/lib"
+addJarInDir "${ZEPPELIN_HOME}/lib/interpreter"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
+
+ZEPPELIN_CLASSPATH="$CLASSPATH:$ZEPPELIN_CLASSPATH"
+
+exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:${ZEPPELIN_CLASSPATH} $MAIN_CLASS "$@"

docs/setup/operation/upgrading.md

@@ -35,6 +35,12 @@ So, copying `notebook` and `conf` directory should be enough.
 
 ## Migration Guide
 
+### Upgrading from Zeppelin 0.8 to 0.9
+
+ - From 0.9, we change the notes file name structure ([ZEPPELIN-2619](https://issues.apache.org/jira/browse/ZEPPELIN-2619)). So when you upgrading zeppelin to 0.9, you need to upgrade note file. Here's steps you need to follow:
+   1. Backup your notes file in case the upgrade fails
+   2. Call `bin/upgrade-note.sh -d` to upgrade note, `-d` option means to delete the old note file, missing this option will keep the old file.  
+
 ### Upgrading from Zeppelin 0.7 to 0.8
 
  - From 0.8, we recommend to use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` instead of `zeppelin.pyspark.python` as `zeppelin.pyspark.python` only effects driver. You can use `PYSPARK_PYTHON` and `PYSPARK_DRIVER_PYTHON` as using them in spark.

zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java

@@ -796,8 +796,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage",
         "org.apache.zeppelin.notebook.repo.GitNotebookRepo"),
     ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC("zeppelin.notebook.one.way.sync", false),
-    ZEPPELIN_NOTEBOOK_NEW_FORMAT_CONVERT("zeppelin.notebook.new_format.convert", false),
-    ZEPPELIN_NOTEBOOK_NEW_FORMAT_DELETE_OLD("zeppelin.notebook.new_format.delete_old", false),
     // whether by default note is public or private
     ZEPPELIN_NOTEBOOK_PUBLIC("zeppelin.notebook.public", true),
     ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner",

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java

@@ -38,6 +38,7 @@ import org.apache.zeppelin.notebook.utility.IdHashes;
 import org.apache.zeppelin.scheduler.Job.Status;
 import org.apache.zeppelin.user.AuthenticationInfo;
 import org.apache.zeppelin.user.Credentials;
+import org.apache.zeppelin.util.Util;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -69,6 +70,7 @@ public class Note implements JsonSerializable {
   private String name = "";
   private String id;
   private String defaultInterpreterGroup;
+  private String version;
   private Map<String, Object> noteParams = new LinkedHashMap<>();
   private Map<String, Input> noteForms = new LinkedHashMap<>();
   private Map<String, List<AngularObject>> angularObjects = new HashMap<>();
@@ -109,6 +111,7 @@ public class Note implements JsonSerializable {
     this.paragraphJobListener = paragraphJobListener;
     this.noteEventListeners = noteEventListener;
     this.credentials = credentials;
+    this.version = Util.getVersion();
     generateId();
 
     setCronSupported(ZeppelinConfiguration.create());
@@ -195,6 +198,10 @@ public class Note implements JsonSerializable {
     this.name = getName(path);
   }
 
+  public void setVersion(String version) {
+    this.version = version;
+  }
+
   public String getDefaultInterpreterGroup() {
     if (defaultInterpreterGroup == null) {
       defaultInterpreterGroup = ZeppelinConfiguration.create()

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java

@@ -28,6 +28,7 @@ import org.apache.zeppelin.notebook.OldNoteInfo;
 import org.apache.zeppelin.notebook.Paragraph;
 import org.apache.zeppelin.plugin.PluginManager;
 import org.apache.zeppelin.user.AuthenticationInfo;
+import org.apache.zeppelin.util.Util;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,7 +45,7 @@ public class NotebookRepoSync implements NotebookRepoWithVersionControl {
   private static final String pullKey = "pullNoteIds";
   private static final String delDstKey = "delDstNoteIds";
 
-  private static final String defaultStorage = "org.apache.zeppelin.notebook.repo.GitNotebookRepo";
+  private static final String DEFAULT_STORAGE = "org.apache.zeppelin.notebook.repo.GitNotebookRepo";
 
   private List<NotebookRepo> repos = new ArrayList<>();
   private boolean oneWaySync;
@@ -62,9 +63,9 @@ public class NotebookRepoSync implements NotebookRepoWithVersionControl {
     oneWaySync = conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC);
     String allStorageClassNames = conf.getNotebookStorageClass().trim();
     if (allStorageClassNames.isEmpty()) {
-      allStorageClassNames = defaultStorage;
+      allStorageClassNames = DEFAULT_STORAGE;
       LOGGER.warn("Empty ZEPPELIN_NOTEBOOK_STORAGE conf parameter, using default {}",
-          defaultStorage);
+              DEFAULT_STORAGE);
     }
     String[] storageClassNames = allStorageClassNames.split(",");
     if (storageClassNames.length > getMaxRepoNum()) {
@@ -72,6 +73,7 @@ public class NotebookRepoSync implements NotebookRepoWithVersionControl {
           "first {} will be used", storageClassNames.length, allStorageClassNames, getMaxRepoNum());
     }
 
+    // init the underlying NotebookRepo
     for (int i = 0; i < Math.min(storageClassNames.length, getMaxRepoNum()); i++) {
       NotebookRepo notebookRepo = PluginManager.get().loadNotebookRepo(storageClassNames[i].trim());
       if (notebookRepo != null) {
@@ -79,40 +81,64 @@ public class NotebookRepoSync implements NotebookRepoWithVersionControl {
         repos.add(notebookRepo);
       }
     }
+
     // couldn't initialize any storage, use default
     if (getRepoCount() == 0) {
-      LOGGER.info("No storage could be initialized, using default {} storage", defaultStorage);
-      NotebookRepo defaultNotebookRepo = PluginManager.get().loadNotebookRepo(defaultStorage);
+      LOGGER.info("No storage could be initialized, using default {} storage", DEFAULT_STORAGE);
+      NotebookRepo defaultNotebookRepo = PluginManager.get().loadNotebookRepo(DEFAULT_STORAGE);
       defaultNotebookRepo.init(conf);
       repos.add(defaultNotebookRepo);
     }
 
-    // convert old note file (noteId/note.json) to new note file (note_name_note_id.zpln)
-    boolean convertToNew = conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_NEW_FORMAT_CONVERT);
-    boolean deleteOld = conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_NEW_FORMAT_DELETE_OLD);
-    if (convertToNew) {
-      NotebookRepo newNotebookRepo = repos.get(0);
-      OldNotebookRepo oldNotebookRepo =
-          PluginManager.get().loadOldNotebookRepo(newNotebookRepo.getClass().getCanonicalName());
-      oldNotebookRepo.init(conf);
-      List<OldNoteInfo> oldNotesInfo = oldNotebookRepo.list(AuthenticationInfo.ANONYMOUS);
-      LOGGER.info("Convert old note file to new style, note count: " + oldNotesInfo.size());
-      for (OldNoteInfo oldNoteInfo : oldNotesInfo) {
-        Note note = oldNotebookRepo.get(oldNoteInfo.getId(), AuthenticationInfo.ANONYMOUS);
-        note.setPath(note.getName());
-        newNotebookRepo.save(note, AuthenticationInfo.ANONYMOUS);
-        if (deleteOld) {
-          oldNotebookRepo.remove(note.getId(), AuthenticationInfo.ANONYMOUS);
-        }
-      }
-    }
-
     // sync for anonymous mode on start
     if (getRepoCount() > 1 && conf.getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED)) {
       try {
         sync(AuthenticationInfo.ANONYMOUS);
       } catch (IOException e) {
-        LOGGER.error("Couldn't sync on start ", e);
+        LOGGER.error("Couldn't sync anonymous mode on start ", e);
+      }
+    }
+  }
+
+  // Zeppelin change its note file name structure in 0.9.0, this is called when upgrading
+  // from 0.9.0 before to 0.9.0 after
+  public void convertNoteFiles(ZeppelinConfiguration conf, boolean deleteOld) throws IOException {
+    // convert old note file (noteId/note.json) to new note file (note_name_note_id.zpln)
+    for (int i = 0; i < repos.size(); ++i) {
+      NotebookRepo newNotebookRepo = repos.get(i);
+      OldNotebookRepo oldNotebookRepo =
+              PluginManager.get().loadOldNotebookRepo(newNotebookRepo.getClass().getCanonicalName());
+      oldNotebookRepo.init(conf);
+      List<OldNoteInfo> oldNotesInfo = oldNotebookRepo.list(AuthenticationInfo.ANONYMOUS);
+      LOGGER.info("Convert old note file to new style, note count: " + oldNotesInfo.size());
+      LOGGER.info("Delete old note: " + deleteOld);
+      for (OldNoteInfo oldNoteInfo : oldNotesInfo) {
+        Note note = oldNotebookRepo.get(oldNoteInfo.getId(), AuthenticationInfo.ANONYMOUS);
+        note.setPath(note.getName());
+        note.setVersion(Util.getVersion());
+        newNotebookRepo.save(note, AuthenticationInfo.ANONYMOUS);
+        if (newNotebookRepo instanceof NotebookRepoWithVersionControl) {
+          ((NotebookRepoWithVersionControl) newNotebookRepo).checkpoint(
+                  note.getId(),
+                  note.getPath(),
+                  "Upgrade note '" + note.getName() + "' to " + Util.getVersion(),
+                  AuthenticationInfo.ANONYMOUS);
+        }
+        if (deleteOld) {
+          oldNotebookRepo.remove(note.getId(), AuthenticationInfo.ANONYMOUS);
+          LOGGER.info("Remote old note: " + note.getId());
+          // TODO(zjffdu) no commit when deleting note, This is an issue of
+          // NotebookRepoWithVersionControl
+          /**
+          if (oldNotebookRepo instanceof NotebookRepoWithVersionControl) {
+            ((NotebookRepoWithVersionControl) oldNotebookRepo).checkpoint(
+                    note.getId(),
+                    note.getName(),
+                    "Delete note '" + note.getName() + "' during note upgrade",
+                    AuthenticationInfo.ANONYMOUS);
+          }
+           **/
+        }
       }
     }
   }

zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/UpgradeNoteFileTool.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import org.apache.commons.cli.*;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+
+import java.io.IOException;
+
+public class UpgradeNoteFileTool {
+
+  public static void main(String[] args) throws IOException {
+    Options options = new Options();
+    Option input = new Option("d", "deleteOld", false, "Whether delete old note file");
+    options.addOption(input);
+    CommandLineParser parser = new DefaultParser();
+    CommandLine cmd = null;
+    try {
+      cmd = parser.parse(options, args);
+    } catch (ParseException e) {
+      System.out.println(e);
+      System.exit(1);
+    }
+
+    ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+    NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
+    notebookRepoSync.convertNoteFiles(conf, cmd.hasOption("d"));
+  }
+}