Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[ZEPPELIN-2214] Soft-code Node, NPM and YarnPkg download/installer urls for Helium module

Browse source
This commit is contained in:
Nelson Costa 2017-05-28 10:45:58 +01:00
parent 87480056a0
commit 6d2a52e4d4
7 changed files with 81 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
conf/zeppelin-env.cmd.template
View file

@ -39,7 +39,9 @@ REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zep
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading
REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading
REM set ZEPPELIN_HELIUM_NPM_REGISTRY REM Remote Npm registry for Helium dependency loader
REM set ZEPPELIN_HELIUM_NODE_INSTALLER_URL REM Remote Node installer url for Helium dependency loader
REM set ZEPPELIN_HELIUM_NPM_INSTALLER_URL REM Remote Npm installer url for Helium dependency loader
REM set ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL REM Remote Yarn package installer url for Helium dependency loader
REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth?

4
conf/zeppelin-env.sh.template
View file

@ -44,7 +44,9 @@
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading
# export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading
# export ZEPPELIN_HELIUM_NPM_REGISTRY # Remote Npm registry for Helium dependency loader
# export ZEPPELIN_HELIUM_NODE_INSTALLER_URL # Remote Node installer url for Helium dependency loader
# export ZEPPELIN_HELIUM_NPM_INSTALLER_URL # Remote Npm installer url for Helium dependency loader
# export ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL # Remote Yarn package installer url for Helium dependency loader
# export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote).
# export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth?
# export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise

16
conf/zeppelin-site.xml.template
View file

@ -252,9 +252,21 @@
</property>
<property>
<name>zeppelin.helium.npm.registry</name>
<name>zeppelin.helium.node.installer.url</name>
<value>https://nodejs.org/dist/</value>
<description>Remote Node installer url for Helium dependency loader</description>
</property>
<property>
<name>zeppelin.helium.npm.installer.url</name>
<value>http://registry.npmjs.org/</value>
<description>Remote Npm registry for Helium dependency loader</description>
<description>Remote Npm installer url for Helium dependency loader</description>
</property>
<property>
<name>zeppelin.helium.yarnpkg.installer.url</name>
<value>https://github.com/yarnpkg/yarn/releases/download/</value>
<description>Remote Yarn package installer url for Helium dependency loader</description>
</property>
<property>

18
docs/install/configuration.md
View file

@ -282,10 +282,22 @@ If both are defined, then the **environment variables** will take priority.
<td>Local repository for dependency loader.<br>ex)visualiztion modules of npm.</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_HELIUM_NPM_REGISTRY</h6></td>
<td><h6 class="properties">zeppelin.helium.npm.registry</h6></td>
<td><h6 class="properties">ZEPPELIN_HELIUM_NODE_INSTALLER_URL</h6></td>
<td><h6 class="properties">zeppelin.helium.node.installer.url</h6></td>
<td>https://nodejs.org/dist/</td>
<td>Remote Node installer url for Helium dependency loader</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_HELIUM_NPM_INSTALLER_URL</h6></td>
<td><h6 class="properties">zeppelin.helium.npm.installer.url</h6></td>
<td>http://registry.npmjs.org/</td>
<td>Remote Npm registry for Helium dependency loader</td>
<td>Remote Npm installer url for Helium dependency loader</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL</h6></td>
<td><h6 class="properties">zeppelin.helium.yarnpkg.installer.url</h6></td>
<td>https://github.com/yarnpkg/yarn/releases/download/</td>
<td>Remote Yarn package installer url for Helium dependency loader</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</h6></td>

19
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -424,8 +424,16 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
}
public String getHeliumNpmRegistry() {
return getString(ConfVars.ZEPPELIN_HELIUM_NPM_REGISTRY);
public String getHeliumNodeInstallerUrl() {
return getString(ConfVars.ZEPPELIN_HELIUM_NODE_INSTALLER_URL);
}
public String getHeliumNpmInstallerUrl() {
return getString(ConfVars.ZEPPELIN_HELIUM_NPM_INSTALLER_URL);
}
public String getHeliumYarnInstallerUrl() {
return getString(ConfVars.ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL);
}
public String getNotebookAuthorizationPath() {
@ -643,7 +651,12 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL),
ZEPPELIN_HELIUM_NPM_REGISTRY("zeppelin.helium.npm.registry", "http://registry.npmjs.org/"),
ZEPPELIN_HELIUM_NODE_INSTALLER_URL("zeppelin.helium.node.installer.url",
"https://nodejs.org/dist/"),
ZEPPELIN_HELIUM_NPM_INSTALLER_URL("zeppelin.helium.npm.installer.url",
"http://registry.npmjs.org/"),
ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL("zeppelin.helium.yarnpkg.installer.url",
"https://github.com/yarnpkg/yarn/releases/download/"),
// Allows a way to specify a ',' separated list of allowed origins for rest and websockets
// i.e. http://localhost:8080
ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),

63
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
View file

@ -49,23 +49,23 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration;
*/
public class HeliumBundleFactory {
Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class);
private final String NODE_VERSION = "v6.9.1";
private final String NPM_VERSION = "3.10.8";
private final String YARN_VERSION = "v0.21.3";
private static final String NODE_VERSION = "v6.9.1";
private static final String NPM_VERSION = "3.10.8";
private static final String YARN_VERSION = "v0.21.3";
private static final String NPM_PACKAGE_NAME = "npm";
public static final String HELIUM_LOCAL_REPO = "helium-bundle";
public static final String HELIUM_BUNDLES_DIR = "bundles";
public static final String HELIUM_LOCAL_MODULE_DIR = "local_modules";
public static final String HELIUM_BUNDLES_SRC_DIR = "src";
public static final String HELIUM_BUNDLES_SRC = "load.js";
public static final String YARN_CACHE_DIR = "yarn-cache";
public static final String PACKAGE_JSON = "package.json";
public static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js";
public static final String HELIUM_BUNDLE = "helium.bundle.js";
public static final String HELIUM_BUNDLES_VAR = "heliumBundles";
private final int FETCH_RETRY_COUNT = 2;
private final int FETCH_RETRY_FACTOR_COUNT = 1;
private final int FETCH_RETRY_MIN_TIMEOUT = 5000; // Milliseconds
protected static final String HELIUM_LOCAL_REPO = "helium-bundle";
private static final String HELIUM_BUNDLES_DIR = "bundles";
private static final String HELIUM_LOCAL_MODULE_DIR = "local_modules";
private static final String HELIUM_BUNDLES_SRC_DIR = "src";
private static final String HELIUM_BUNDLES_SRC = "load.js";
private static final String YARN_CACHE_DIR = "yarn-cache";
private static final String PACKAGE_JSON = "package.json";
private static final String HELIUM_BUNDLE_CACHE = "helium.bundle.cache.js";
private static final String HELIUM_BUNDLE = "helium.bundle.js";
private static final String HELIUM_BUNDLES_VAR = "heliumBundles";
private static final int FETCH_RETRY_COUNT = 2;
private static final int FETCH_RETRY_FACTOR_COUNT = 1;
private static final int FETCH_RETRY_MIN_TIMEOUT = 5000; // Milliseconds
private final FrontendPluginFactory frontEndPluginFactory;
private final File nodeInstallationDirectory;
@ -73,13 +73,12 @@ public class HeliumBundleFactory {
private final File heliumBundleDirectory;
private final File heliumLocalModuleDirectory;
private final File yarnCacheDir;
private ZeppelinConfiguration conf;
private File tabledataModulePath;
private File visualizationModulePath;
private File spellModulePath;
private String defaultNodeRegistryUrl;
private String defaultNpmRegistryUrl;
private String defaultYarnRegistryUrl;
private String defaultNodeInstallerUrl;
private String defaultNpmInstallerUrl;
private String defaultYarnInstallerUrl;
private Gson gson;
private boolean nodeAndNpmInstalled = false;
@ -106,11 +105,9 @@ public class HeliumBundleFactory {
this.heliumBundleDirectory = new File(heliumLocalRepoDirectory, HELIUM_BUNDLES_DIR);
this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR);
this.yarnCacheDir = new File(heliumLocalRepoDirectory, YARN_CACHE_DIR);
this.conf = conf;
// To be done in ZEPPELIN-2214: Soft-code installer urls
this.defaultNodeRegistryUrl = "https://nodejs.org/dist/";
this.defaultNpmRegistryUrl = conf.getHeliumNpmRegistry();
this.defaultYarnRegistryUrl = "https://github.com/yarnpkg/yarn/releases/download/";
this.defaultNodeInstallerUrl = conf.getHeliumNodeInstallerUrl();
this.defaultNpmInstallerUrl = conf.getHeliumNpmInstallerUrl();
this.defaultYarnInstallerUrl = conf.getHeliumYarnInstallerUrl();
nodeInstallationDirectory = (nodeInstallationDir == null) ?
heliumLocalRepoDirectory : nodeInstallationDir;
@ -127,21 +124,21 @@ public class HeliumBundleFactory {
}
try {
NodeInstaller nodeInstaller = frontEndPluginFactory
.getNodeInstaller(getProxyConfig(isSecure(defaultNodeRegistryUrl)));
.getNodeInstaller(getProxyConfig(isSecure(defaultNodeInstallerUrl)));
nodeInstaller.setNodeVersion(NODE_VERSION);
nodeInstaller.setNodeDownloadRoot(defaultNodeRegistryUrl);
nodeInstaller.setNodeDownloadRoot(defaultNodeInstallerUrl);
nodeInstaller.install();
NPMInstaller npmInstaller = frontEndPluginFactory
.getNPMInstaller(getProxyConfig(isSecure(defaultNpmRegistryUrl)));
.getNPMInstaller(getProxyConfig(isSecure(defaultNpmInstallerUrl)));
npmInstaller.setNpmVersion(NPM_VERSION);
npmInstaller.setNpmDownloadRoot(defaultNpmRegistryUrl + "/" + NPM_PACKAGE_NAME + "/-/");
npmInstaller.setNpmDownloadRoot(defaultNpmInstallerUrl + "/" + NPM_PACKAGE_NAME + "/-/");
npmInstaller.install();
YarnInstaller yarnInstaller = frontEndPluginFactory
.getYarnInstaller(getProxyConfig(isSecure(defaultYarnRegistryUrl)));
.getYarnInstaller(getProxyConfig(isSecure(defaultYarnInstallerUrl)));
yarnInstaller.setYarnVersion(YARN_VERSION);
yarnInstaller.setYarnDownloadRoot(defaultYarnRegistryUrl);
yarnInstaller.setYarnDownloadRoot(defaultYarnInstallerUrl);
yarnInstaller.install();
yarnCacheDir.mkdirs();
String yarnCacheDirPath = yarnCacheDir.getAbsolutePath();
@ -663,7 +660,7 @@ public class HeliumBundleFactory {
private void npmCommand(String args, Map<String, String> env) throws TaskRunnerException {
NpmRunner npm = frontEndPluginFactory.getNpmRunner(
getProxyConfig(isSecure(defaultNpmRegistryUrl)), defaultNpmRegistryUrl);
getProxyConfig(isSecure(defaultNpmInstallerUrl)), defaultNpmInstallerUrl);
npm.execute(args, env);
}
@ -678,7 +675,7 @@ public class HeliumBundleFactory {
private void yarnCommand(FrontendPluginFactory fpf,
String args, Map<String, String> env) throws TaskRunnerException {
YarnRunner yarn = fpf.getYarnRunner(
getProxyConfig(isSecure(defaultNpmRegistryUrl)), defaultNpmRegistryUrl);
getProxyConfig(isSecure(defaultNpmInstallerUrl)), defaultNpmInstallerUrl);
yarn.execute(args, env);
}

1
zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
View file

@ -119,7 +119,6 @@ public class HeliumBundleFactoryTest {
assertEquals(lastModified, bundle.lastModified());
}
@Test
public void bundleLocalPackage() throws IOException, TaskRunnerException {
URL res = Resources.getResource("helium/webpack.config.js");