diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md index 57bd6afc97..5825d1d629 100644 --- a/docs/manual/interpreterinstallation.md +++ b/docs/manual/interpreterinstallation.md @@ -111,9 +111,9 @@ You can also install 3rd party interpreters located in the maven repository by u ./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1 ``` -The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. +The above command will download maven artifact `groupId1:artifact1:version1` and all of its transitive dependencies into `interpreter/interpreter1` directory. -After restart Zeppelin, then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). +After restart Zeppelin, then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your note](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). #### Install multiple 3rd party interpreters at once diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 30ff4bcbf8..f1c5676c6d 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -38,6 +38,7 @@ import org.apache.spark.SparkContext; import org.apache.spark.SparkEnv; import org.apache.spark.SecurityManager; +import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.repl.SparkILoop; import org.apache.spark.scheduler.ActiveJob; import org.apache.spark.scheduler.DAGScheduler; @@ -126,6 +127,7 @@ public class SparkInterpreter extends Interpreter { private SparkVersion sparkVersion; private static File outputDir; // class outputdir for scala 2.11 private Object classServer; // classserver for scala 2.11 + private JavaSparkContext jsc; public SparkInterpreter(Properties property) { @@ -152,6 +154,15 @@ public class SparkInterpreter extends Interpreter { } } + public JavaSparkContext getJavaSparkContext() { + synchronized (sharedInterpreterLock) { + if (jsc == null) { + jsc = JavaSparkContext.fromSparkContext(sc); + } + return jsc; + } + } + public boolean isSparkContextInitialized() { synchronized (sharedInterpreterLock) { return sc != null; @@ -1422,6 +1433,7 @@ public class SparkInterpreter extends Interpreter { } sparkSession = null; sc = null; + jsc = null; if (classServer != null) { Utils.invokeMethod(classServer, "stop"); classServer = null; diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java index 16b1a21144..75687d05d8 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.spark.SparkContext; import org.apache.spark.SparkRBackend; +import org.apache.spark.api.java.JavaSparkContext; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.scheduler.Scheduler; @@ -45,6 +46,7 @@ public class SparkRInterpreter extends Interpreter { private SparkInterpreter sparkInterpreter; private ZeppelinR zeppelinR; private SparkContext sc; + private JavaSparkContext jsc; public SparkRInterpreter(Properties property) { super(property); @@ -73,8 +75,10 @@ public class SparkRInterpreter extends Interpreter { this.sparkInterpreter = getSparkInterpreter(); this.sc = sparkInterpreter.getSparkContext(); + this.jsc = sparkInterpreter.getJavaSparkContext(); SparkVersion sparkVersion = new SparkVersion(sc.version()); ZeppelinRContext.setSparkContext(sc); + ZeppelinRContext.setJavaSparkContext(jsc); if (Utils.isSpark2()) { ZeppelinRContext.setSparkSession(sparkInterpreter.getSparkSession()); } diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java index 935410bdd5..a2fc4120c5 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.spark; import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; /** @@ -28,6 +29,7 @@ public class ZeppelinRContext { private static SQLContext sqlContext; private static ZeppelinContext zeppelinContext; private static Object sparkSession; + private static JavaSparkContext javaSparkContext; public static void setSparkContext(SparkContext sparkContext) { ZeppelinRContext.sparkContext = sparkContext; @@ -60,4 +62,8 @@ public class ZeppelinRContext { public static Object getSparkSession() { return sparkSession; } + + public static void setJavaSparkContext(JavaSparkContext jsc) { javaSparkContext = jsc; } + + public static JavaSparkContext getJavaSparkContext() { return javaSparkContext; } } diff --git a/spark/src/main/resources/R/zeppelin_sparkr.R b/spark/src/main/resources/R/zeppelin_sparkr.R index e95513f745..525c6c5c40 100644 --- a/spark/src/main/resources/R/zeppelin_sparkr.R +++ b/spark/src/main/resources/R/zeppelin_sparkr.R @@ -45,7 +45,7 @@ assign("sc", get(".sc", envir = SparkR:::.sparkREnv), envir=.GlobalEnv) if (version >= 20000) { assign(".sparkRsession", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkSession"), envir = SparkR:::.sparkREnv) assign("spark", get(".sparkRsession", envir = SparkR:::.sparkREnv), envir = .GlobalEnv) - assign(".sparkRjsc", get(".sc", envir = SparkR:::.sparkREnv), envir=SparkR:::.sparkREnv) + assign(".sparkRjsc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getJavaSparkContext"), envir = SparkR:::.sparkREnv) } assign(".sqlc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSqlContext"), envir = SparkR:::.sparkREnv) assign("sqlContext", get(".sqlc", envir = SparkR:::.sparkREnv), envir = .GlobalEnv) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java index 62c4bcf49e..c1af75fc88 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java @@ -34,6 +34,10 @@ public class HeliumPackage { // [[ .. and .. and .. ] or [ .. and .. and ..] ..] private String license; private String icon; + private String published; + + private String groupId; // get groupId of INTERPRETER type package + private String artifactId; // get artifactId of INTERPRETER type package private SpellPackageInfo spell; private Map config; @@ -108,6 +112,18 @@ public class HeliumPackage { return icon; } + public String getPublishedDate() { + return published; + } + + public String getGroupId() { + return groupId; + } + + public String getArtifactId() { + return artifactId; + } + public SpellPackageInfo getSpellInfo() { return spell; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java index 76d90b9877..0e9dbebaec 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java @@ -182,22 +182,16 @@ public abstract class Job { this.exception = null; errorMessage = null; dateFinished = new Date(); - progressUpdator.terminate(); - } catch (NullPointerException e) { - LOGGER.error("Job failed", e); - progressUpdator.terminate(); - this.exception = e; - setResult(e.getMessage()); - errorMessage = getStack(e); - dateFinished = new Date(); } catch (Throwable e) { LOGGER.error("Job failed", e); - progressUpdator.terminate(); this.exception = e; setResult(e.getMessage()); errorMessage = getStack(e); dateFinished = new Date(); } finally { + if (progressUpdator != null) { + progressUpdator.interrupt(); + } //aborted = false; } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java index 967702a63c..8b8cda0af8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java @@ -21,48 +21,45 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** + * Polls job progress with given interval + * + * @see Job#progress() + * @see JobListener#onProgressUpdate(org.apache.zeppelin.scheduler.Job, int) + * * TODO(moon) : add description. */ public class JobProgressPoller extends Thread { public static final long DEFAULT_INTERVAL_MSEC = 500; - Logger logger = LoggerFactory.getLogger(JobProgressPoller.class); + private static final Logger logger = LoggerFactory.getLogger(JobProgressPoller.class); + private Job job; private long intervalMs; - boolean terminate = false; public JobProgressPoller(Job job, long intervalMs) { + super("JobProgressPoller, jobId=" + job.getId()); this.job = job; - this.intervalMs = intervalMs; + if (intervalMs < 0) { + throw new IllegalArgumentException("polling interval can't be " + intervalMs); + } + this.intervalMs = intervalMs == 0 ? DEFAULT_INTERVAL_MSEC : intervalMs; } @Override public void run() { - if (intervalMs < 0) { - return; - } else if (intervalMs == 0) { - intervalMs = DEFAULT_INTERVAL_MSEC; - } - - while (terminate == false) { - JobListener listener = job.getListener(); - if (listener != null) { - try { - if (job.isRunning()) { - listener.onProgressUpdate(job, job.progress()); + try { + while (!Thread.interrupted()) { + JobListener listener = job.getListener(); + if (listener != null) { + try { + if (job.isRunning()) { + listener.onProgressUpdate(job, job.progress()); + } + } catch (Exception e) { + logger.error("Can not get or update progress", e); } - } catch (Exception e) { - logger.error("Can not get or update progress", e); } - } - try { Thread.sleep(intervalMs); - } catch (InterruptedException e) { - logger.error("Exception in JobProgressPoller while run Thread.sleep", e); } - } - } - - public void terminate() { - terminate = true; + } catch (InterruptedException ignored) {} } } diff --git a/zeppelin-web/src/app/helium/helium.controller.js b/zeppelin-web/src/app/helium/helium.controller.js index 5a19ea0cac..3819133afb 100644 --- a/zeppelin-web/src/app/helium/helium.controller.js +++ b/zeppelin-web/src/app/helium/helium.controller.js @@ -23,7 +23,15 @@ export default function HeliumCtrl($scope, $rootScope, $sce, $scope.showVersions = {}; $scope.bundleOrder = []; $scope.bundleOrderChanged = false; + $scope.vizTypePkg = {} + $scope.spellTypePkg = {} + $scope.intpTypePkg = {} + $scope.appTypePkg = {} + $scope.numberOfEachPackageByType = {} + $scope.allPackageTypes = [HeliumType][0] + $scope.pkgListByType = 'VISUALIZATION' $scope.defaultPackageConfigs = {}; // { pkgName, [{name, type, desc, value, defaultValue}] } + $scope.intpDefaultIcon = $sce.trustAsHtml(''); function init() { // get all package info and set config @@ -31,6 +39,7 @@ export default function HeliumCtrl($scope, $rootScope, $sce, .then(({ pkgSearchResults, defaultPackages }) => { $scope.pkgSearchResults = pkgSearchResults; $scope.defaultPackages = defaultPackages; + classifyPkgType($scope.defaultPackages) return heliumService.getAllPackageConfigs() }) .then(defaultPackageConfigs => { @@ -45,6 +54,38 @@ export default function HeliumCtrl($scope, $rootScope, $sce, }); } + var classifyPkgType = function(packageInfos) { + var vizTypePkg = {} + var spellTypePkg = {} + var intpTypePkg = {} + var appTypePkg = {} + + for (var name in packageInfos) { + var pkgs = packageInfos[name] + var pkgType = pkgs.pkg.type + + switch (pkgType) { + case HeliumType.VISUALIZATION: + vizTypePkg[name] = pkgs; + break; + case HeliumType.SPELL: + spellTypePkg[name] = pkgs; + break; + case HeliumType.INTERPRETER: + intpTypePkg[name] = pkgs; + break; + case HeliumType.APPLICATION: + appTypePkg[name] = pkgs; + break; + } + } + + $scope.vizTypePkg = vizTypePkg + $scope.spellTypePkg = spellTypePkg + $scope.appTypePkg = appTypePkg + $scope.intpTypePkg = intpTypePkg + }; + $scope.bundleOrderListeners = { accept: function(sourceItemHandleScope, destSortableScope) {return true;}, itemMoved: function(event) {}, @@ -108,40 +149,58 @@ export default function HeliumCtrl($scope, $rootScope, $sce, return license; } - $scope.enable = function(name, artifact) { + $scope.enable = function(name, artifact, type, groupId) { var license = getLicense(name, artifact); - - var confirm = BootstrapDialog.confirm({ - closable: false, - closeByBackdrop: false, - closeByKeyboard: false, + var mavenArtifactInfoToHTML = groupId +':'+ artifact.split('@')[0] + ':' + artifact.split('@')[1]; + var zeppelinVersion = $rootScope.zeppelinVersion; + var url = 'https://zeppelin.apache.org/docs/' + zeppelinVersion + '/manual/interpreterinstallation.html'; + + var confirm = '' + if (type === 'INTERPRETER') { + confirm = BootstrapDialog.show({ title: '', - message: 'Do you want to enable ' + name + '?' + - '
' + artifact + '
' + - '
License
' + - '
' + license + '
', - callback: function(result) { - if (result) { - confirm.$modalFooter.find('button').addClass('disabled'); - confirm.$modalFooter.find('button:contains("OK")') - .html(' Enabling'); - heliumService.enable(name, artifact). - success(function(data, status) { - init(); - confirm.close(); - }). - error(function(data, status) { - confirm.close(); - console.log('Failed to enable package %o %o. %o', name, artifact, data); - BootstrapDialog.show({ - title: 'Error on enabling ' + name, - message: data.message - }); - }); - return false; - } - } + message: '

Below command will download maven artifact ' + + '' + + mavenArtifactInfoToHTML + '' + + ' and all of its transitive dependencies into interpreter/interpreter-name directory.

' + + '

' +
+      './bin/install-interpreter.sh --name "interpreter-name" --artifact ' +
+        mavenArtifactInfoToHTML +'
' + + '

After restart Zeppelin, create interpreter setting and bind it with your note. ' + + 'For more detailed information, see Interpreter Installation.

' }); + } else { + confirm = BootstrapDialog.confirm({ + closable: false, + closeByBackdrop: false, + closeByKeyboard: false, + title: '', + message: 'Do you want to enable ' + name + '?' + + '
' + artifact + '
' + + '
License
' + + '
' + license + '
', + callback: function (result) { + if (result) { + confirm.$modalFooter.find('button').addClass('disabled'); + confirm.$modalFooter.find('button:contains("OK")') + .html(' Enabling'); + heliumService.enable(name, artifact, type).success(function (data, status) { + init(); + confirm.close(); + }).error(function (data, status) { + confirm.close(); + console.log('Failed to enable package %o %o. %o', name, artifact, data); + BootstrapDialog.show({ + title: 'Error on enabling ' + name, + message: data.message + }); + }); + return false; + } + } + }); + } }; $scope.disable = function(name) { @@ -194,6 +253,20 @@ export default function HeliumCtrl($scope, $rootScope, $sce, !$scope.isLocalPackage(pkgSearchResult); }; + $scope.hasMavenLink = function(pkgSearchResult) { + const pkg = pkgSearchResult.pkg; + return (pkg.type === HeliumType.APPLICATION || pkg.type === HeliumType.INTERPRETER) && + !$scope.isLocalPackage(pkgSearchResult); + }; + + $scope.getPackageSize = function(pkgSearchResult, targetPkgType) { + var result = [] + _.map(pkgSearchResult, function (pkg) { + result.push(_.find(pkg, {type: targetPkgType})) + }) + return _.compact(result).length + } + $scope.configExists = function(pkgSearchResult) { // helium package config is persisted per version return pkgSearchResult.pkg.config && pkgSearchResult.pkg.artifact; diff --git a/zeppelin-web/src/app/helium/helium.css b/zeppelin-web/src/app/helium/helium.css index 23e8c19f2f..63a81ff78f 100644 --- a/zeppelin-web/src/app/helium/helium.css +++ b/zeppelin-web/src/app/helium/helium.css @@ -136,6 +136,33 @@ color: #636363; } +.heliumLearnMore { + margin-top:10px; +} + +.heliumLearnMore a { + cursor:pointer; + margin-right:10px; + text-decoration:none; +} + +.heliumRepoBtn { + margin-right: 8px; +} + +.heliumRepoBtn:hover, .heliumRepoBtn:focus { + margin-right: 8px; + outline: 0; +} + +.localPkgInfo { + margin: 10px 12px 0 0; + font-size: 11px; + font-style: italic; + color: #aaaaaa; + text-align: right; +} + .heliumConfig { margin-top: 30px; margin-bottom: 10px; @@ -147,10 +174,6 @@ margin-bottom: 15px; } -.heliumConfigValueInput { - -} - .heliumConfigValueText { vertical-align: top; } diff --git a/zeppelin-web/src/app/helium/helium.html b/zeppelin-web/src/app/helium/helium.html index 7718666477..b64f71cc31 100644 --- a/zeppelin-web/src/app/helium/helium.html +++ b/zeppelin-web/src/app/helium/helium.html @@ -18,6 +18,23 @@ limitations under the License.

Helium

+
+ + + + +

* Local registry package's name is gray colored.

+
+ ng-show="bundleOrderChanged" + ng-click="saveBundleOrder()"> save @@ -44,28 +61,46 @@ limitations under the License.
+
+
+ Currently there is no available package to be listed +
+
- + ng-repeat="pkgSearchResult in defaultPackages | toArray:false | orderBy: 'pkg.published':true" + ng-show="$parent.pkgListByType === pkgSearchResult.pkg.type">
+
- {{pkgName}} + {{pkgSearchResult.pkg.name}} - - {{pkgName}} + + {{pkgSearchResult.pkg.name}} + + + + {{pkgSearchResult.pkg.name}} + {{pkgSearchResult.pkg.type}}
Enable
Disable
{{pkgSearchResult.pkg.artifact}} - + versions
    + ng-show="showVersions[pkgSearchResult.pkg.name]">
  • + ng-repeat="pkgSearchResult in pkgSearchResults[pkgSearchResult.pkg.name]"> {{pkgSearchResult.pkg.artifact}} - - enable + + see more +
diff --git a/zeppelin-web/src/assets/images/maven_default_icon.png b/zeppelin-web/src/assets/images/maven_default_icon.png new file mode 100644 index 0000000000..56b38947b2 Binary files /dev/null and b/zeppelin-web/src/assets/images/maven_default_icon.png differ diff --git a/zeppelin-web/src/components/helium/helium-type.js b/zeppelin-web/src/components/helium/helium-type.js index 0ef4eb67a6..27b34fa696 100644 --- a/zeppelin-web/src/components/helium/helium-type.js +++ b/zeppelin-web/src/components/helium/helium-type.js @@ -15,4 +15,6 @@ export const HeliumType = { VISUALIZATION: 'VISUALIZATION', SPELL: 'SPELL', + INTERPRETER: 'INTERPRETER', + APPLICATION: 'APPLICATION', } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java index 62f8e02082..de03195af3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java @@ -34,7 +34,7 @@ import java.net.URL; import java.util.*; /** - * Load helium visualization + * Load helium visualization & spell */ public class HeliumBundleFactory { Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class);