implement online registry

zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java

@@ -123,7 +123,7 @@ public class ZeppelinServer extends Application {
 
     this.helium = new Helium(
         conf.getHeliumConfPath(),
-        conf.getHeliumDefaultLocalRegistryPath(),
+        conf.getHeliumRegistry(),
         heliumVisualizationFactory,
         heliumApplicationFactory);
 

zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java

@@ -41,6 +41,9 @@ public class ZeppelinConfiguration extends XMLConfiguration {
   private static final String ZEPPELIN_SITE_XML = "zeppelin-site.xml";
   private static final long serialVersionUID = 4749305895693848035L;
   private static final Logger LOG = LoggerFactory.getLogger(ZeppelinConfiguration.class);
+
+  private static final String HELIUM_PACKAGE_DEFAULT_URL =
+      "https://s3.amazonaws.com/helium-package/helium.json";
   private static ZeppelinConfiguration conf;
 
   public ZeppelinConfiguration(URL url) throws ConfigurationException {
@@ -397,8 +400,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     return getRelativeDir(String.format("%s/helium.json", getConfDir()));
   }
 
-  public String getHeliumDefaultLocalRegistryPath() {
-    return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_LOCALREGISTRY_DEFAULT);
+  public String getHeliumRegistry() {
+    return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
   }
 
   public String getNotebookAuthorizationPath() {
@@ -599,7 +602,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
     ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true),
     ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
     ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
-    ZEPPELIN_HELIUM_LOCALREGISTRY_DEFAULT("zeppelin.helium.localregistry.default", "helium"),
+    ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL),
     // Allows a way to specify a ',' separated list of allowed origins for rest and websockets
     // i.e. http://localhost:8080
     ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java

@@ -41,19 +41,19 @@ public class Helium {
 
   private final HeliumConf heliumConf;
   private final String heliumConfPath;
-  private final String defaultLocalRegistryPath;
+  private final String registryPaths;
   private final Gson gson;
   private final HeliumVisualizationFactory visualizationFactory;
   private final HeliumApplicationFactory applicationFactory;
 
   public Helium(
       String heliumConfPath,
-      String defaultLocalRegistryPath,
+      String registryPaths,
       HeliumVisualizationFactory visualizationFactory,
       HeliumApplicationFactory applicationFactory)
       throws IOException {
     this.heliumConfPath = heliumConfPath;
-    this.defaultLocalRegistryPath = defaultLocalRegistryPath;
+    this.registryPaths = registryPaths;
     this.visualizationFactory = visualizationFactory;
     this.applicationFactory = applicationFactory;
 
@@ -96,19 +96,28 @@ public class Helium {
   }
 
   private synchronized HeliumConf loadConf(String path) throws IOException {
+    // add registry
+    if (registryPaths != null && !registryPaths.isEmpty()) {
+      String[] paths = registryPaths.split(",");
+      for (String uri : paths) {
+        if (uri.startsWith("http://") || uri.startsWith("https://")) {
+          logger.info("Add helium online registry {}", uri);
+          registry.add(new HeliumOnlineRegistry(uri, uri));
+        } else {
+          logger.info("Add helium local registry {}", uri);
+          registry.add(new HeliumLocalRegistry(uri, uri));
+        }
+      }
+    }
+
     File heliumConfFile = new File(path);
     if (!heliumConfFile.isFile()) {
       logger.warn("{} does not exists", path);
       HeliumConf conf = new HeliumConf();
-      LinkedList<HeliumRegistry> defaultRegistry = new LinkedList<>();
-      defaultRegistry.add(new HeliumLocalRegistry("local", defaultLocalRegistryPath));
-      conf.setRegistry(defaultRegistry);
-      this.registry = conf.getRegistry();
       return conf;
     } else {
       String jsonString = FileUtils.readFileToString(heliumConfFile);
       HeliumConf conf = gson.fromJson(jsonString, HeliumConf.class);
-      this.registry = conf.getRegistry();
       return conf;
     }
   }
@@ -117,7 +126,6 @@ public class Helium {
     String jsonString;
     synchronized (registry) {
       clearNotExistsPackages();
-      heliumConf.setRegistry(registry);
       jsonString = gson.toJson(heliumConf);
     }
 

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java

@@ -22,23 +22,12 @@ import java.util.*;
  * Helium config. This object will be persisted to conf/heliumc.conf
  */
 public class HeliumConf {
-  List<HeliumRegistry> registry = new LinkedList<>();
-
   // enabled packages {name, version}
   Map<String, String> enabled = Collections.synchronizedMap(new HashMap<String, String>());
 
   // enabled visualization package display order
   List<String> visualizationDisplayOrder = new LinkedList<>();
 
-
-  public List<HeliumRegistry> getRegistry() {
-    return registry;
-  }
-
-  public void setRegistry(List<HeliumRegistry> registry) {
-    this.registry = registry;
-  }
-
   public Map<String, String> getEnabledPackages() {
     return new HashMap<>(enabled);
   }

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumOnlineRegistry.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.helium;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This registry reads helium package json data
+ * from specified url.
+ *
+ * File should be look like
+ * [
+ *    "packageName": {
+ *       "0.0.1": json serialized HeliumPackage class,
+ *       "0.0.2": json serialized HeliumPackage class,
+ *       ...
+ *    },
+ *    ...
+ * ]
+ */
+public class HeliumOnlineRegistry extends HeliumRegistry {
+  Logger logger = LoggerFactory.getLogger(HeliumOnlineRegistry.class);
+  private final Gson gson;
+
+  public HeliumOnlineRegistry(String name, String uri) {
+    super(name, uri);
+    gson = new Gson();
+  }
+
+  @Override
+  public List<HeliumPackage> getAll() throws IOException {
+    BufferedReader reader = new BufferedReader(new InputStreamReader(new URL(uri()).openStream()));
+    List<Map<String, Map<String, HeliumPackage>>> packages = gson.fromJson(
+        reader,
+        new TypeToken<List<Map<String, Map<String, HeliumPackage>>>>() {
+        }.getType());
+    reader.close();
+
+    List<HeliumPackage> packageList = new LinkedList<>();
+
+    for (Map<String, Map<String, HeliumPackage>> pkg : packages) {
+      for (Map<String, HeliumPackage> versions : pkg.values()) {
+        packageList.addAll(versions.values());
+      }
+    }
+
+    return packageList;
+  }
+}