feat: Loading bundles

zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java

@@ -24,6 +24,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.zeppelin.helium.Helium;
 import org.apache.zeppelin.helium.HeliumPackage;
+import org.apache.zeppelin.helium.HeliumPackageSearchResult;
 import org.apache.zeppelin.notebook.Note;
 import org.apache.zeppelin.notebook.Notebook;
 import org.apache.zeppelin.notebook.Paragraph;
@@ -68,6 +69,16 @@ public class HeliumRestApi {
         Response.Status.OK, "", helium.getAllPackageInfo()).build();
   }
 
+  /**
+   * Get all enabled package infos
+   */
+  @GET
+  @Path("enabledPackage")
+  public Response getAllEnabledPackageInfo() {
+    return new JsonResponse(
+            Response.Status.OK, "", helium.getAllEnabledPackages()).build();
+  }
+
   /**
    * Get single package info
    */
@@ -130,26 +141,43 @@ public class HeliumRestApi {
   }
 
   @GET
-  @Path("bundle/load")
+  @Path("bundle/load/{packageName}")
   @Produces("text/javascript")
-  public Response bundleLoad(@QueryParam("refresh") String refresh) {
+  public Response bundleLoad(@QueryParam("refresh") String refresh,
+                             @PathParam("packageName") String packageName) {
+    if (StringUtils.isEmpty(packageName)) {
+      return new JsonResponse(
+          Response.Status.BAD_REQUEST,
+          "Can't get bundle due to empty package name").build();
+    }
+
+    HeliumPackageSearchResult psr = null;
+    List<HeliumPackageSearchResult> enabledPackages = helium.getAllEnabledPackages();
+    for (HeliumPackageSearchResult e : enabledPackages) {
+      if (e.getPkg().getName().equals(packageName)) {
+        psr = e;
+        break;
+      }
+    }
+
+    if (psr == null) {
+      // return empty to specify
+      return Response.ok().build();
+    }
+
     try {
       File bundle;
-      if (refresh != null && refresh.equals("true")) {
-        bundle = helium.recreateBundle();
-      } else {
-        bundle = helium.getBundleFactory().getCurrentCacheBundle();
-      }
+      boolean rebuild = (refresh != null && refresh.equals("true"));
+      bundle = helium.getBundle(psr.getPkg(), rebuild);
 
       if (bundle == null) {
         return Response.ok().build();
       } else {
-        String stringifiedBundle = FileUtils.readFileToString(bundle);
-        return Response.ok(stringifiedBundle).build();
+        String stringified = FileUtils.readFileToString(bundle);
+        return Response.ok(stringified).build();
       }
     } catch (Exception e) {
       logger.error(e.getMessage(), e);
-
       // returning error will prevent zeppelin front-end render any notebook.
       // visualization load fail doesn't need to block notebook rendering work.
       // so it's better return ok instead of any error.

zeppelin-web/src/components/helium/helium.service.js

@@ -27,41 +27,14 @@ angular.module('zeppelinWebApp').service('heliumService', heliumService);
 export default function heliumService($http, $sce, baseUrlSrv) {
   'ngInject';
 
-  var url = baseUrlSrv.getRestApiBase() + '/helium/bundle/load';
-  if (process.env.HELIUM_BUNDLE_DEV) {
-    url = url + '?refresh=true';
-  }
-
   let visualizationBundles = [];
-  // name `heliumBundles` should be same as `HelumBundleFactory.HELIUM_BUNDLES_VAR`
+  // name `heliumBundles` should be same as `HeliumBundleFactory.HELIUM_BUNDLES_VAR`
   let heliumBundles = [];
   // map for `{ magic: interpreter }`
   let spellPerMagic = {};
   // map for `{ magic: package-name }`
   let pkgNamePerMagic = {}
 
-  // load should be promise
-  this.load = $http.get(url).success(function(response) {
-    if (response.substring(0, 'ERROR:'.length) !== 'ERROR:') {
-      // evaluate bundles
-      eval(response);
-
-      // extract bundles by type
-      heliumBundles.map(b => {
-        if (b.type === HeliumType.SPELL) {
-          const spell = new b.class(); // eslint-disable-line new-cap
-          const pkgName = b.id;
-          spellPerMagic[spell.getMagic()] = spell;
-          pkgNamePerMagic[spell.getMagic()] = pkgName;
-        } else if (b.type === HeliumType.VISUALIZATION) {
-          visualizationBundles.push(b);
-        }
-      });
-    } else {
-      console.error(response);
-    }
-  });
-
   /**
    * @param magic {string} e.g `%flowchart`
    * @returns {SpellBase} undefined if magic is not registered
@@ -160,6 +133,31 @@ export default function heliumService($http, $sce, baseUrlSrv) {
       });
   };
 
+  this.getAllEnabledPackages = function() {
+    return $http.get(`${baseUrlSrv.getRestApiBase()}/helium/enabledPackage`)
+      .then(function(response, status) {
+        return response.data.body;
+      })
+      .catch(function(error) {
+        console.error('Failed to get all enabled package infos', error);
+      });
+  };
+
+  this.getSingleBundle = function(pkgName) {
+    let url = `${baseUrlSrv.getRestApiBase()}/helium/bundle/load/${pkgName}`
+    if (process.env.HELIUM_BUNDLE_DEV) {
+      url = url + '?refresh=true';
+    }
+
+    return $http.get(url)
+      .then(function(response, status) {
+        return response.data;
+      })
+      .catch(function(error) {
+        console.error(`Failed to get single bundle: ${pkgName}`, error);
+      });
+  }
+
   this.getDefaultPackages = function() {
     return this.getAllPackageInfo()
       .then(pkgSearchResults => {
@@ -241,4 +239,48 @@ export default function heliumService($http, $sce, baseUrlSrv) {
       return merged;
     });
   }
+
+  const p = this.getAllEnabledPackages()
+    .then(enabledPackageSearchResults => {
+      const promises = enabledPackageSearchResults.map(packageSearchResult => {
+        const pkgName = packageSearchResult.pkg.name
+        return this.getSingleBundle(pkgName)
+      })
+
+      return Promise.all(promises)
+    })
+    .then(bundles => {
+      return bundles.reduce((acc, b) => {
+        // filter out empty bundle
+        if (b === '') { return acc }
+        acc.push(b)
+        return acc;
+      }, [])
+    })
+
+  // load should be promise
+  this.load = p.then(availableBundles => {
+    // if (response.substring(0, 'ERROR:'.length) === 'ERROR:') {
+    //   console.error(response);
+    //   return
+    // }
+    
+    // evaluate bundles
+    availableBundles.map(b => {
+      eval(b)
+    })
+
+    // extract bundles by type
+    heliumBundles.map(b => {
+      if (b.type === HeliumType.SPELL) {
+        const spell = new b.class() // eslint-disable-line new-cap
+        const pkgName = b.id
+        spellPerMagic[spell.getMagic()] = spell
+        pkgNamePerMagic[spell.getMagic()] = pkgName
+      } else if (b.type === HeliumType.VISUALIZATION) {
+        visualizationBundles.push(b)
+      }
+    })
+  })
+
 }

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java

@@ -242,6 +242,22 @@ public class Helium {
     }
   }
 
+  public List<HeliumPackageSearchResult> getAllEnabledPackages() {
+    Map<String, List<HeliumPackageSearchResult>> allPackages = getAllPackageInfoWithoutRefresh();
+    List<HeliumPackageSearchResult> enabledPackages = new ArrayList<>();
+
+    for (List<HeliumPackageSearchResult> versionedPackages : allPackages.values()) {
+      for (HeliumPackageSearchResult psr : versionedPackages) {
+        if (psr.isEnabled()) {
+          enabledPackages.add(psr);
+          break;
+        }
+      }
+    }
+
+    return enabledPackages;
+  }
+
   public List<HeliumPackageSearchResult> getSinglePackageInfo(String packageName) {
     Map<String, List<HeliumPackageSearchResult>> result = getAllPackageInfo(false, packageName);
 
@@ -281,8 +297,8 @@ public class Helium {
     return null;
   }
 
-  public File recreateBundle() throws IOException {
-    return bundleFactory.buildAllPackages(getBundlePackagesToBundle(), true);
+  public File getBundle(HeliumPackage pkg, boolean rebuild) throws IOException {
+    return bundleFactory.buildPackage(pkg, rebuild);
   }
 
   public void enable(String name, String artifact) throws IOException {
@@ -295,7 +311,7 @@ public class Helium {
 
     // if package is visualization, rebuild bundle
     if (HeliumPackage.isBundleType(pkgInfo.getPkg().getType())) {
-      bundleFactory.buildPackage(pkgInfo.getPkg());
+      bundleFactory.buildPackage(pkgInfo.getPkg(), true);
     }
 
     // update conf and save

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java

@@ -143,16 +143,16 @@ public class HeliumBundleFactory {
     return buildAllPackages(pkgs, false);
   }
 
-  public File getHeliumPackageDirectory(String artifact) {
-    return new File(heliumBundleDirectory, artifact);
+  public File getHeliumPackageDirectory(String pkgName) {
+    return new File(heliumBundleDirectory, pkgName);
   }
 
-  public File getHeliumPackageSourceDirectory(String artifact) {
-    return new File(heliumBundleDirectory, artifact + "/" + HELIUM_BUNDLES_SRC_DIR);
+  public File getHeliumPackageSourceDirectory(String pkgName) {
+    return new File(heliumBundleDirectory, pkgName + "/" + HELIUM_BUNDLES_SRC_DIR);
   }
 
-  public File getHeliumPackageBundleCache(String artifact) {
-    return new File(heliumBundleDirectory, artifact + "/" + HELIUM_BUNDLE_CACHE);
+  public File getHeliumPackageBundleCache(String pkgName) {
+    return new File(heliumBundleDirectory, pkgName + "/" + HELIUM_BUNDLE_CACHE);
   }
 
   public static List<String> unTgz(File tarFile, File directory) throws IOException {
@@ -181,9 +181,12 @@ public class HeliumBundleFactory {
     return result;
   }
 
-  public void downloadPackage(HeliumPackage pkg, String[] nameAndVersion, File bundleDir,
-                              String templateWebpackConfig, String templatePackageJson,
-                              FrontendPluginFactory fpf) throws IOException, TaskRunnerException {
+  /**
+   * @return main file name of this helium package (relative path)
+   */
+  public String downloadPackage(HeliumPackage pkg, String[] nameAndVersion, File bundleDir,
+                                String templateWebpackConfig, String templatePackageJson,
+                                FrontendPluginFactory fpf) throws IOException, TaskRunnerException {
     if (bundleDir.exists()) {
       FileUtils.deleteQuietly(bundleDir);
     }
@@ -230,13 +233,6 @@ public class HeliumBundleFactory {
     Map<String, String> existingDeps = (Map<String, String>) packageJson.get("dependencies");
     String mainFileName = (String) packageJson.get("main");
 
-    // package.json doesn't require postfix `.js`, but webpack needs it.
-    if (!mainFileName.endsWith(".js") &&
-        !(new File(bundleDir, mainFileName).exists()) &&
-        (new File(bundleDir, mainFileName + ".js").exists())) {
-      mainFileName = mainFileName + ".js";
-    }
-
     StringBuilder dependencies = new StringBuilder();
     int index = 0;
     for (Map.Entry<String, String> e: existingDeps.entrySet()) {
@@ -263,16 +259,26 @@ public class HeliumBundleFactory {
     FileUtils.write(new File(bundleDir, PACKAGE_JSON), templatePackageJson);
 
     // 2. setup webpack.config
-    templateWebpackConfig = templateWebpackConfig.replaceFirst("MAIN_FILE", "./" + mainFileName);
     FileUtils.write(new File(bundleDir, "webpack.config.js"), templateWebpackConfig);
+
+    return mainFileName;
   }
 
-  public void prepareSource(HeliumPackage pkg, String[] moduleNameVersion) throws IOException {
+  public void prepareSource(HeliumPackage pkg, String[] moduleNameVersion,
+                            String mainFileName) throws IOException {
     StringBuilder loadJsImport = new StringBuilder();
     StringBuilder loadJsRegister = new StringBuilder();
     String className = "bundles" + pkg.getName().replaceAll("[-_]", "");
-    loadJsImport.append(
-            "import " + className + " from \"" + moduleNameVersion[0] + "\"\n");
+
+    // remove postfix `.js` for ES6 import
+    if (mainFileName.endsWith(".js")) {
+      mainFileName = mainFileName.substring(0, mainFileName.length() - 2);
+    }
+
+    loadJsImport
+        .append("import ")
+        .append(className)
+        .append(" from \"../" + mainFileName + "\"\n");
 
     loadJsRegister.append(HELIUM_BUNDLES_VAR + ".push({\n");
     loadJsRegister.append("id: \"" + moduleNameVersion[0] + "\",\n");
@@ -326,53 +332,60 @@ public class HeliumBundleFactory {
     return heliumBundle;
   }
 
-  public synchronized void buildPackage(HeliumPackage pkg) throws IOException {
-    // resources: webpack.js, package.json
-    String templateWebpackConfig = Resources.toString(
-            Resources.getResource("helium/webpack.config.js"), Charsets.UTF_8);
-    String templatePackageJson = Resources.toString(
-            Resources.getResource("helium/" + PACKAGE_JSON), Charsets.UTF_8);
-
+  public synchronized File buildPackage(HeliumPackage pkg, boolean rebuild) throws IOException {
     if (pkg == null) {
-      return;
+      return null;
     }
 
     String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
     if (moduleNameVersion == null) {
-      return;
+      return null;
     }
 
     if (moduleNameVersion == null) {
       logger.error("Can't get module name and version of package " + pkg.getName());
-      return;
+      return null;
     }
+
     String pkgName = pkg.getName();
+    File bundleDir = getHeliumPackageDirectory(pkgName);
+    File bundleCache = getHeliumPackageBundleCache(pkgName);
+
+    if (!rebuild && bundleCache.exists() && !bundleCache.isDirectory()) {
+      return bundleCache;
+    }
 
     // 0. prepare directory
-    File bundleDir = getHeliumPackageDirectory(pkgName);
     FrontendPluginFactory fpf = new FrontendPluginFactory(
             bundleDir, heliumLocalRepoDirectory);
 
+    // resources: webpack.js, package.json
+    String templateWebpackConfig = Resources.toString(
+        Resources.getResource("helium/webpack.config.js"), Charsets.UTF_8);
+    String templatePackageJson = Resources.toString(
+        Resources.getResource("helium/" + PACKAGE_JSON), Charsets.UTF_8);
+
     // 1. download project
+    String mainFileName = null;
     try {
-      downloadPackage(pkg, moduleNameVersion, bundleDir,
+      mainFileName = downloadPackage(pkg, moduleNameVersion, bundleDir,
               templateWebpackConfig, templatePackageJson, fpf);
     } catch (TaskRunnerException e) {
       throw new IOException(e);
     }
 
     // 2. prepare bundle source
-    prepareSource(pkg, moduleNameVersion);
+    prepareSource(pkg, moduleNameVersion, mainFileName);
 
     // 3. install npm modules for a bundle
     installNodeModules(fpf);
 
     // 4. let's bundle and update cache
     File heliumBundle = bundleHeliumPackage(fpf, bundleDir);
+    bundleCache.delete();
+    FileUtils.moveFile(heliumBundle, bundleCache);
 
-    File cache = getHeliumPackageBundleCache(pkgName);
-    cache.delete();
-    FileUtils.moveFile(heliumBundle, cache);
+    return bundleCache;
   }
 
   public synchronized File buildAllPackages(List<HeliumPackage> pkgs, boolean forceRefresh)
@@ -393,7 +406,7 @@ public class HeliumBundleFactory {
     copyFrameworkModuleToInstallPath();
 
     for (HeliumPackage pkg : pkgs) {
-      buildPackage(pkg);
+      buildPackage(pkg, forceRefresh);
     }
 
     return currentCacheBundle;

zeppelin-zengine/src/main/resources/helium/webpack.config.js

@@ -16,7 +16,7 @@
  */
 
 module.exports = {
-    entry: 'MAIN_FILE',
+    entry: './src/load.js',
     output: { path: './', filename: 'helium.bundle.js', },
   module: {
     loaders: [