bundle visualization package from npm repository on the fly

zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java

@@ -37,7 +37,8 @@ public class HeliumPackage {
   public static enum Type {
     INTERPRETER,
     NOTEBOOK_REPO,
-    APPLICATION
+    APPLICATION,
+    VISUALIZATION
   }
 
   public HeliumPackage(Type type,

zeppelin-web/src/app/tabledata/.npmignore

@@ -0,0 +1 @@
+*.html

zeppelin-web/src/app/tabledata/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "zeppelin-tabledata",
+  "description": "tabledata api",
+  "version": "0.7.0-SNAPSHOT",
+  "dependencies": {
+    "json3": "~3.3.1",
+    "lodash": "~3.9.3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/apache/zeppelin"
+  },
+  "license": "Apache-2.0"
+}

zeppelin-web/src/app/visualization/.npmignore

@@ -0,0 +1 @@
+*.html

zeppelin-web/src/app/visualization/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "zeppelin-vis",
+  "description": "Visualization API",
+  "version": "0.7.0-SNAPSHOT",
+  "dependencies": {
+    "json3": "~3.3.1",
+    "nvd3": "~1.7.1",
+    "lodash": "~3.9.3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/apache/zeppelin"
+  },
+  "license": "Apache-2.0"
+}

zeppelin-zengine/pom.xml

@@ -45,6 +45,7 @@
     <org.reflections.version>0.9.8</org.reflections.version>
     <xml.apis.version>1.4.01</xml.apis.version>
     <eclipse.jgit.version>4.1.1.201511131810-r</eclipse.jgit.version>
+    <frontend.maven.plugin.version>1.3</frontend.maven.plugin.version>
 
     <!--test library versions-->
     <google.truth.version>0.27</google.truth.version>
@@ -217,6 +218,12 @@
       <version>${eclipse.jgit.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>com.github.eirslett</groupId>
+      <artifactId>frontend-maven-plugin</artifactId>
+      <version>${frontend.maven.plugin.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -256,7 +263,6 @@
       <version>${jetty.version}</version>
       <scope>test</scope>
     </dependency>
-
   </dependencies>
 
   <build>

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumVisualizationFactory.java

@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.helium;
+
+import com.github.eirslett.maven.plugins.frontend.lib.*;
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
+import com.google.gson.Gson;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Load helium visualization
+ */
+public class HeliumVisualizationFactory {
+  Logger logger = LoggerFactory.getLogger(HeliumVisualizationFactory.class);
+  private final String NODE_VERSION = "v6.9.1";
+  private final String NPM_VERSION = "3.10.8";
+  private final String DEFAULT_NPM_REGISTRY_URL = "http://registry.npmjs.org/";
+
+  private final FrontendPluginFactory frontEndPluginFactory;
+  private final File workingDirectory;
+  private File tabledataModulePath;
+  private File visualizationModulePath;
+
+  public HeliumVisualizationFactory(
+      File moduleDownloadPath,
+      File tabledataModulePath,
+      File visualizationModulePath) throws InstallationException, TaskRunnerException {
+    this(moduleDownloadPath);
+    this.tabledataModulePath = tabledataModulePath;
+    this.visualizationModulePath = visualizationModulePath;
+  }
+
+  public HeliumVisualizationFactory(File moduleDownloadPath)
+      throws InstallationException, TaskRunnerException {
+    this.workingDirectory = moduleDownloadPath;
+    File installDirectory = moduleDownloadPath;
+
+    frontEndPluginFactory = new FrontendPluginFactory(
+        workingDirectory, installDirectory);
+
+    installNodeAndNpm();
+  }
+
+  private void installNodeAndNpm() throws InstallationException, TaskRunnerException {
+    NPMInstaller npmInstaller = frontEndPluginFactory.getNPMInstaller(getProxyConfig());
+    npmInstaller.setNpmVersion(NPM_VERSION);
+    npmInstaller.install();
+
+    NodeInstaller nodeInstaller = frontEndPluginFactory.getNodeInstaller(getProxyConfig());
+    nodeInstaller.setNodeVersion(NODE_VERSION);
+    nodeInstaller.install();
+  }
+
+  private ProxyConfig getProxyConfig() {
+    List<ProxyConfig.Proxy> proxy = new LinkedList<>();
+    return new ProxyConfig(proxy);
+  }
+
+  public void bundle(List<HeliumPackage> pkgs) throws IOException, TaskRunnerException {
+    // package.json
+    URL pkgUrl = Resources.getResource("helium/package.json");
+    String pkgJson = Resources.toString(pkgUrl, Charsets.UTF_8);
+    StringBuffer dependencies = new StringBuffer();
+    for (HeliumPackage pkg : pkgs) {
+      String[] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
+      if (moduleNameVersion == null) {
+        logger.error("Can't get module name and version of package " + pkg.getName());
+        continue;
+      }
+      if (dependencies.length() > 0) {
+        dependencies.append(",\n");
+      }
+      dependencies.append("\"" + moduleNameVersion[0] + "\": \"" + moduleNameVersion[1] + "\"");
+
+      if (isLocalPackage(pkg)) {
+        FileUtils.copyDirectory(new File(pkg.getArtifact()),
+            new File(workingDirectory, "node_modules/" + pkg.getName()));
+      }
+    }
+    pkgJson = pkgJson.replaceFirst("DEPENDENCIES", dependencies.toString());
+
+    // webpack.config.js
+    URL webpackConfigUrl = Resources.getResource("helium/webpack.config.js");
+    String webpackConfig = Resources.toString(webpackConfigUrl, Charsets.UTF_8);
+
+    // generate load.js
+    StringBuilder loadJs = new StringBuilder();
+    for (HeliumPackage pkg : pkgs) {
+      String [] moduleNameVersion = getNpmModuleNameAndVersion(pkg);
+      if (moduleNameVersion == null) {
+        continue;
+      }
+      loadJs.append("import " + moduleNameVersion[0] + " from \"" + moduleNameVersion[0] + "\"\n");
+    }
+
+    FileUtils.write(new File(workingDirectory, "package.json"), pkgJson);
+    FileUtils.write(new File(workingDirectory, "webpack.config.js"), webpackConfig);
+    FileUtils.write(new File(workingDirectory, "load.js"), loadJs.toString());
+
+    // install tabledata module
+    File tabledataModuleInstallPath = new File(workingDirectory,
+        "node_modules/zeppelin-tabledata");
+    if (tabledataModulePath != null && !tabledataModuleInstallPath.exists()) {
+      FileUtils.copyDirectory(tabledataModulePath, tabledataModuleInstallPath);
+    }
+
+    // install visualization module
+    File visModuleInstallPath = new File(workingDirectory,
+        "node_modules/zeppelin-vis");
+    if (visualizationModulePath != null && !visModuleInstallPath.exists()) {
+      FileUtils.copyDirectory(visualizationModulePath, visModuleInstallPath);
+    }
+
+    npmCommand("install");
+    npmCommand("run bundle");
+  }
+
+  private boolean isLocalPackage(HeliumPackage pkg) {
+    return (pkg.getArtifact().startsWith(".") || pkg.getArtifact().startsWith("/"));
+  }
+
+  private String [] getNpmModuleNameAndVersion(HeliumPackage pkg) {
+    String artifact = pkg.getArtifact();
+
+    if (isLocalPackage(pkg)) {
+      File packageJson = new File(artifact, "package.json");
+      if (!packageJson.isFile()) {
+        return null;
+      }
+      Gson gson = new Gson();
+      try {
+        NpmPackage npmPackage = gson.fromJson(
+            FileUtils.readFileToString(packageJson),
+            NpmPackage.class);
+
+        String[] nameVersion = new String[2];
+        nameVersion[0] = npmPackage.name;
+        nameVersion[1] = npmPackage.version;
+        return nameVersion;
+      } catch (IOException e) {
+        logger.error(e.getMessage(), e);
+        return null;
+      }
+    } else {
+      String[] nameVersion = new String[2];
+
+      int pos;
+      if ((pos = artifact.indexOf('@')) > 0) {
+        nameVersion[0] = artifact.substring(0, pos);
+        nameVersion[1] = artifact.substring(pos + 1);
+      } else if (
+          (pos = artifact.indexOf('^')) > 0 ||
+              (pos = artifact.indexOf('~')) > 0) {
+        nameVersion[0] = artifact.substring(0, pos);
+        nameVersion[1] = artifact.substring(pos);
+      } else {
+        nameVersion[0] = artifact;
+        nameVersion[1] = "";
+      }
+      return nameVersion;
+    }
+  }
+
+  public synchronized void install(HeliumPackage pkg) throws TaskRunnerException {
+    npmCommand("install " + pkg.getArtifact());
+  }
+
+  private void npmCommand(String args) throws TaskRunnerException {
+    npmCommand(args, new HashMap<String, String>());
+  }
+  private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException {
+    NpmRunner npm = frontEndPluginFactory.getNpmRunner(getProxyConfig(), DEFAULT_NPM_REGISTRY_URL);
+    npm.execute(args, env);
+  }
+}

zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/NpmPackage.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.helium;
+
+import java.util.Map;
+
+/**
+ * To read package.json
+ */
+public class NpmPackage {
+  public String name;
+  public String version;
+  public Map<String, String> dependencies;
+}

zeppelin-zengine/src/main/resources/helium/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "zeppelin-vis-bundle",
+  "main": "load",
+  "scripts": {
+    "bundle": "node/node node_modules/webpack/bin/webpack.js --display-error-details"
+  },
+  "dependencies": {
+    DEPENDENCIES
+  },
+  "devDependencies": {
+    "webpack": "^1.12.2",
+    "babel": "^5.8.23",
+    "babel-loader": "^5.3.2"
+  }
+}

zeppelin-zengine/src/main/resources/helium/webpack.config.js

@@ -0,0 +1,18 @@
+module.exports = {
+    entry: ['./'],
+    output: {
+        path: './',
+        filename: 'vis.bundle.js',
+    },
+    resolve: {
+        root: __dirname + "/node_modules",
+        extensions: [".js"]
+    },
+    module: {
+        loaders: [{
+            test: /\.js$/,
+            //exclude: /node_modules/,
+            loader: 'babel-loader'
+        }]
+    }
+}

zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumVisualizationFactoryTest.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.helium;
+
+import com.github.eirslett.maven.plugins.frontend.lib.InstallationException;
+import com.github.eirslett.maven.plugins.frontend.lib.TaskRunnerException;
+import com.google.common.io.Resources;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.LinkedList;
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+public class HeliumVisualizationFactoryTest {
+  private File tmpDir;
+  private HeliumVisualizationFactory hvf;
+
+  @Before
+  public void setUp() throws InstallationException, TaskRunnerException {
+    //tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis());
+    tmpDir = new File("/tmp/npm");
+    tmpDir.mkdirs();
+
+    // get module dir
+    URL res = Resources.getResource("helium/webpack.config.js");
+    String resDir = new File(res.getFile()).getParent();
+    File moduleDir = new File(resDir + "/../../../../zeppelin-web/src/app/");
+
+    hvf = new HeliumVisualizationFactory(tmpDir,
+        new File(moduleDir, "tabledata"),
+        new File(moduleDir, "visualization"));
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    //FileUtils.deleteDirectory(tmpDir);
+  }
+
+  @Test
+  public void testInstallNpm() throws InstallationException {
+    assertTrue(new File(tmpDir, "node/npm").isFile());
+    assertTrue(new File(tmpDir, "node/node").isFile());
+  }
+
+  @Test
+  public void downloadPackage() throws TaskRunnerException {
+    HeliumPackage pkg = new HeliumPackage(
+        HeliumPackage.Type.VISUALIZATION,
+        "lodash",
+        "lodash",
+        "lodash^3.9.3",
+        "",
+        null
+    );
+    hvf.install(pkg);
+    assertTrue(new File(tmpDir, "node_modules/lodash").isDirectory());
+  }
+
+  @Test
+  public void bundlePackage() throws IOException, TaskRunnerException {
+    HeliumPackage pkg = new HeliumPackage(
+        HeliumPackage.Type.VISUALIZATION,
+        "lodash",
+        "lodash",
+        "lodash^3.9.3",
+        "",
+        null
+    );
+    List<HeliumPackage> pkgs = new LinkedList<>();
+    pkgs.add(pkg);
+    hvf.bundle(pkgs);
+    assertTrue(new File(tmpDir, "vis.bundle.js").isFile());
+  }
+
+
+  @Test
+  public void bundleLocalPackage() throws IOException, TaskRunnerException {
+    URL res = Resources.getResource("helium/webpack.config.js");
+    String resDir = new File(res.getFile()).getParent();
+    String localPkg = resDir + "/../../../src/test/resources/helium/vis1";
+    System.err.println("local package " + localPkg);
+
+    HeliumPackage pkg = new HeliumPackage(
+        HeliumPackage.Type.VISUALIZATION,
+        "vis1",
+        "vis1",
+        localPkg,
+        "",
+        null
+    );
+    List<HeliumPackage> pkgs = new LinkedList<>();
+    pkgs.add(pkg);
+    hvf.bundle(pkgs);
+    assertTrue(new File(tmpDir, "vis.bundle.js").isFile());
+  }
+}

zeppelin-zengine/src/test/resources/helium/vis1/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "vis1",
+  "version": "1.0.0",
+  "description": "",
+  "main": "vis1",
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "zeppelin-tabledata": "0.7.0-SNAPSHOT",
+    "zeppelin-vis": "0.7.0-SNAPSHOT"
+  }
+}

zeppelin-zengine/src/test/resources/helium/vis1/vis1.js

@@ -0,0 +1,2 @@
+
+console.log('vis1');