Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/master' into ZEPPELIN-2149

Browse source
This commit is contained in:
Prabhjyot Singh 2017-03-08 14:13:54 +05:30
commit a989f5e355
14 changed files with 257 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/manual/interpreterinstallation.md
View file

@ -111,9 +111,9 @@ You can also install 3rd party interpreters located in the maven repository by u
./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1
```
The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory.
The above command will download maven artifact `groupId1:artifact1:version1` and all of its transitive dependencies into `interpreter/interpreter1` directory.
After restart Zeppelin, then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
After restart Zeppelin, then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your note](../manual/interpreters.html#what-is-zeppelin-interpreter-setting).
#### Install multiple 3rd party interpreters at once

12
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -38,6 +38,7 @@ import org.apache.spark.SparkContext;
import org.apache.spark.SparkEnv;
import org.apache.spark.SecurityManager;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.repl.SparkILoop;
import org.apache.spark.scheduler.ActiveJob;
import org.apache.spark.scheduler.DAGScheduler;
@ -126,6 +127,7 @@ public class SparkInterpreter extends Interpreter {
private SparkVersion sparkVersion;
private static File outputDir; // class outputdir for scala 2.11
private Object classServer; // classserver for scala 2.11
private JavaSparkContext jsc;
public SparkInterpreter(Properties property) {
@ -152,6 +154,15 @@ public class SparkInterpreter extends Interpreter {
}
}
public JavaSparkContext getJavaSparkContext() {
synchronized (sharedInterpreterLock) {
if (jsc == null) {
jsc = JavaSparkContext.fromSparkContext(sc);
}
return jsc;
}
}
public boolean isSparkContextInitialized() {
synchronized (sharedInterpreterLock) {
return sc != null;
@ -1422,6 +1433,7 @@ public class SparkInterpreter extends Interpreter {
}
sparkSession = null;
sc = null;
jsc = null;
if (classServer != null) {
Utils.invokeMethod(classServer, "stop");
classServer = null;

4
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.spark.SparkContext;
import org.apache.spark.SparkRBackend;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.scheduler.Scheduler;
@ -45,6 +46,7 @@ public class SparkRInterpreter extends Interpreter {
private SparkInterpreter sparkInterpreter;
private ZeppelinR zeppelinR;
private SparkContext sc;
private JavaSparkContext jsc;
public SparkRInterpreter(Properties property) {
super(property);
@ -73,8 +75,10 @@ public class SparkRInterpreter extends Interpreter {
this.sparkInterpreter = getSparkInterpreter();
this.sc = sparkInterpreter.getSparkContext();
this.jsc = sparkInterpreter.getJavaSparkContext();
SparkVersion sparkVersion = new SparkVersion(sc.version());
ZeppelinRContext.setSparkContext(sc);
ZeppelinRContext.setJavaSparkContext(jsc);
if (Utils.isSpark2()) {
ZeppelinRContext.setSparkSession(sparkInterpreter.getSparkSession());
}

6
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinRContext.java
View file

@ -18,6 +18,7 @@
package org.apache.zeppelin.spark;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SQLContext;
/**
@ -28,6 +29,7 @@ public class ZeppelinRContext {
private static SQLContext sqlContext;
private static ZeppelinContext zeppelinContext;
private static Object sparkSession;
private static JavaSparkContext javaSparkContext;
public static void setSparkContext(SparkContext sparkContext) {
ZeppelinRContext.sparkContext = sparkContext;
@ -60,4 +62,8 @@ public class ZeppelinRContext {
public static Object getSparkSession() {
return sparkSession;
}
public static void setJavaSparkContext(JavaSparkContext jsc) { javaSparkContext = jsc; }
public static JavaSparkContext getJavaSparkContext() { return javaSparkContext; }
}

2
spark/src/main/resources/R/zeppelin_sparkr.R
View file

@ -45,7 +45,7 @@ assign("sc", get(".sc", envir = SparkR:::.sparkREnv), envir=.GlobalEnv)
if (version >= 20000) {
assign(".sparkRsession", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkSession"), envir = SparkR:::.sparkREnv)
assign("spark", get(".sparkRsession", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)
assign(".sparkRjsc", get(".sc", envir = SparkR:::.sparkREnv), envir=SparkR:::.sparkREnv)
assign(".sparkRjsc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getJavaSparkContext"), envir = SparkR:::.sparkREnv)
}
assign(".sqlc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSqlContext"), envir = SparkR:::.sparkREnv)
assign("sqlContext", get(".sqlc", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)

16
zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java
View file

@ -34,6 +34,10 @@ public class HeliumPackage {
// [[ .. and .. and .. ] or [ .. and .. and ..] ..]
private String license;
private String icon;
private String published;
private String groupId; // get groupId of INTERPRETER type package
private String artifactId; // get artifactId of INTERPRETER type package
private SpellPackageInfo spell;
private Map<String, Object> config;
@ -108,6 +112,18 @@ public class HeliumPackage {
return icon;
}
public String getPublishedDate() {
return published;
}
public String getGroupId() {
return groupId;
}
public String getArtifactId() {
return artifactId;
}
public SpellPackageInfo getSpellInfo() {
return spell;
}

12
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/Job.java
View file

@ -182,22 +182,16 @@ public abstract class Job {
this.exception = null;
errorMessage = null;
dateFinished = new Date();
progressUpdator.terminate();
} catch (NullPointerException e) {
LOGGER.error("Job failed", e);
progressUpdator.terminate();
this.exception = e;
setResult(e.getMessage());
errorMessage = getStack(e);
dateFinished = new Date();
} catch (Throwable e) {
LOGGER.error("Job failed", e);
progressUpdator.terminate();
this.exception = e;
setResult(e.getMessage());
errorMessage = getStack(e);
dateFinished = new Date();
} finally {
if (progressUpdator != null) {
progressUpdator.interrupt();
}
//aborted = false;
}
}

49
zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/JobProgressPoller.java
View file

@ -21,48 +21,45 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Polls job progress with given interval
*
* @see Job#progress()
* @see JobListener#onProgressUpdate(org.apache.zeppelin.scheduler.Job, int)
*
* TODO(moon) : add description.
*/
public class JobProgressPoller extends Thread {
public static final long DEFAULT_INTERVAL_MSEC = 500;
Logger logger = LoggerFactory.getLogger(JobProgressPoller.class);
private static final Logger logger = LoggerFactory.getLogger(JobProgressPoller.class);
private Job job;
private long intervalMs;
boolean terminate = false;
public JobProgressPoller(Job job, long intervalMs) {
super("JobProgressPoller, jobId=" + job.getId());
this.job = job;
this.intervalMs = intervalMs;
if (intervalMs < 0) {
throw new IllegalArgumentException("polling interval can't be " + intervalMs);
}
this.intervalMs = intervalMs == 0 ? DEFAULT_INTERVAL_MSEC : intervalMs;
}
@Override
public void run() {
if (intervalMs < 0) {
return;
} else if (intervalMs == 0) {
intervalMs = DEFAULT_INTERVAL_MSEC;
}
while (terminate == false) {
JobListener listener = job.getListener();
if (listener != null) {
try {
if (job.isRunning()) {
listener.onProgressUpdate(job, job.progress());
try {
while (!Thread.interrupted()) {
JobListener listener = job.getListener();
if (listener != null) {
try {
if (job.isRunning()) {
listener.onProgressUpdate(job, job.progress());
}
} catch (Exception e) {
logger.error("Can not get or update progress", e);
}
} catch (Exception e) {
logger.error("Can not get or update progress", e);
}
}
try {
Thread.sleep(intervalMs);
} catch (InterruptedException e) {
logger.error("Exception in JobProgressPoller while run Thread.sleep", e);
}
}
}
public void terminate() {
terminate = true;
} catch (InterruptedException ignored) {}
}
}

135
zeppelin-web/src/app/helium/helium.controller.js
View file

@ -23,7 +23,15 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
$scope.showVersions = {};
$scope.bundleOrder = [];
$scope.bundleOrderChanged = false;
$scope.vizTypePkg = {}
$scope.spellTypePkg = {}
$scope.intpTypePkg = {}
$scope.appTypePkg = {}
$scope.numberOfEachPackageByType = {}
$scope.allPackageTypes = [HeliumType][0]
$scope.pkgListByType = 'VISUALIZATION'
$scope.defaultPackageConfigs = {}; // { pkgName, [{name, type, desc, value, defaultValue}] }
$scope.intpDefaultIcon = $sce.trustAsHtml('<img src="../assets/images/maven_default_icon.png" style="width: 12px"/>');
function init() {
// get all package info and set config
@ -31,6 +39,7 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
.then(({ pkgSearchResults, defaultPackages }) => {
$scope.pkgSearchResults = pkgSearchResults;
$scope.defaultPackages = defaultPackages;
classifyPkgType($scope.defaultPackages)
return heliumService.getAllPackageConfigs()
})
.then(defaultPackageConfigs => {
@ -45,6 +54,38 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
});
}
var classifyPkgType = function(packageInfos) {
var vizTypePkg = {}
var spellTypePkg = {}
var intpTypePkg = {}
var appTypePkg = {}
for (var name in packageInfos) {
var pkgs = packageInfos[name]
var pkgType = pkgs.pkg.type
switch (pkgType) {
case HeliumType.VISUALIZATION:
vizTypePkg[name] = pkgs;
break;
case HeliumType.SPELL:
spellTypePkg[name] = pkgs;
break;
case HeliumType.INTERPRETER:
intpTypePkg[name] = pkgs;
break;
case HeliumType.APPLICATION:
appTypePkg[name] = pkgs;
break;
}
}
$scope.vizTypePkg = vizTypePkg
$scope.spellTypePkg = spellTypePkg
$scope.appTypePkg = appTypePkg
$scope.intpTypePkg = intpTypePkg
};
$scope.bundleOrderListeners = {
accept: function(sourceItemHandleScope, destSortableScope) {return true;},
itemMoved: function(event) {},
@ -108,40 +149,58 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
return license;
}
$scope.enable = function(name, artifact) {
$scope.enable = function(name, artifact, type, groupId) {
var license = getLicense(name, artifact);
var confirm = BootstrapDialog.confirm({
closable: false,
closeByBackdrop: false,
closeByKeyboard: false,
var mavenArtifactInfoToHTML = groupId +':'+ artifact.split('@')[0] + ':' + artifact.split('@')[1];
var zeppelinVersion = $rootScope.zeppelinVersion;
var url = 'https://zeppelin.apache.org/docs/' + zeppelinVersion + '/manual/interpreterinstallation.html';
var confirm = ''
if (type === 'INTERPRETER') {
confirm = BootstrapDialog.show({
title: '',
message: 'Do you want to enable ' + name + '?' +
'<div style="color:gray">' + artifact + '</div>' +
'<div style="border-top: 1px solid #efefef; margin-top: 10px; padding-top: 5px;">License</div>' +
'<div style="color:gray">' + license + '</div>',
callback: function(result) {
if (result) {
confirm.$modalFooter.find('button').addClass('disabled');
confirm.$modalFooter.find('button:contains("OK")')
.html('<i class="fa fa-circle-o-notch fa-spin"></i> Enabling');
heliumService.enable(name, artifact).
success(function(data, status) {
init();
confirm.close();
}).
error(function(data, status) {
confirm.close();
console.log('Failed to enable package %o %o. %o', name, artifact, data);
BootstrapDialog.show({
title: 'Error on enabling ' + name,
message: data.message
});
});
return false;
}
}
message: '<p>Below command will download maven artifact ' +
'<code style="font-size: 11.5px; background-color: #f5f5f5; color: #0a0a0a">' +
mavenArtifactInfoToHTML + '</code>' +
' and all of its transitive dependencies into interpreter/interpreter-name directory.<p>' +
'<div class="highlight"><pre><code class="text language-text" data-lang="text" style="font-size: 11.5px">' +
'./bin/install-interpreter.sh --name "interpreter-name" --artifact ' +
mavenArtifactInfoToHTML +' </code></pre>' +
'<p>After restart Zeppelin, create interpreter setting and bind it with your note. ' +
'For more detailed information, see <a target="_blank" href=' +
url + '>Interpreter Installation.</a></p>'
});
} else {
confirm = BootstrapDialog.confirm({
closable: false,
closeByBackdrop: false,
closeByKeyboard: false,
title: '',
message: 'Do you want to enable ' + name + '?' +
'<div style="color:gray">' + artifact + '</div>' +
'<div style="border-top: 1px solid #efefef; margin-top: 10px; padding-top: 5px;">License</div>' +
'<div style="color:gray">' + license + '</div>',
callback: function (result) {
if (result) {
confirm.$modalFooter.find('button').addClass('disabled');
confirm.$modalFooter.find('button:contains("OK")')
.html('<i class="fa fa-circle-o-notch fa-spin"></i> Enabling');
heliumService.enable(name, artifact, type).success(function (data, status) {
init();
confirm.close();
}).error(function (data, status) {
confirm.close();
console.log('Failed to enable package %o %o. %o', name, artifact, data);
BootstrapDialog.show({
title: 'Error on enabling ' + name,
message: data.message
});
});
return false;
}
}
});
}
};
$scope.disable = function(name) {
@ -194,6 +253,20 @@ export default function HeliumCtrl($scope, $rootScope, $sce,
!$scope.isLocalPackage(pkgSearchResult);
};
$scope.hasMavenLink = function(pkgSearchResult) {
const pkg = pkgSearchResult.pkg;
return (pkg.type === HeliumType.APPLICATION || pkg.type === HeliumType.INTERPRETER) &&
!$scope.isLocalPackage(pkgSearchResult);
};
$scope.getPackageSize = function(pkgSearchResult, targetPkgType) {
var result = []
_.map(pkgSearchResult, function (pkg) {
result.push(_.find(pkg, {type: targetPkgType}))
})
return _.compact(result).length
}
$scope.configExists = function(pkgSearchResult) {
// helium package config is persisted per version
return pkgSearchResult.pkg.config && pkgSearchResult.pkg.artifact;

31
zeppelin-web/src/app/helium/helium.css
View file

@ -136,6 +136,33 @@
color: #636363;
}
.heliumLearnMore {
margin-top:10px;
}
.heliumLearnMore a {
cursor:pointer;
margin-right:10px;
text-decoration:none;
}
.heliumRepoBtn {
margin-right: 8px;
}
.heliumRepoBtn:hover, .heliumRepoBtn:focus {
margin-right: 8px;
outline: 0;
}
.localPkgInfo {
margin: 10px 12px 0 0;
font-size: 11px;
font-style: italic;
color: #aaaaaa;
text-align: right;
}
.heliumConfig {
margin-top: 30px;
margin-bottom: 10px;
@ -147,10 +174,6 @@
margin-bottom: 15px;
}
.heliumConfigValueInput {
}
.heliumConfigValueText {
vertical-align: top;
}

70
zeppelin-web/src/app/helium/helium.html
View file

@ -18,6 +18,23 @@ limitations under the License.
<h3 class="new_h3">
Helium
</h3>
<div class="pull-right heliumLearnMore">
<a target="_blank"
class="helium-repo-btn"
ng-href="https://zeppelin.apache.org/helium_packages.html"
tooltip-placement="bottom"
tooltip="Learn more">
<i class="icon-question" ng-style="{color: 'black'}"></i>
</a>
<button tabindex="0" class="btn btn-default btn-sm heliumRepoBtn helium-popover"
role="button"
ng-repeat="pkgTypes in allPackageTypes"
ng-click="$parent.pkgListByType = pkgTypes">
<i class="fa fa-cube"></i>
{{pkgTypes}}
</button>
<p class="localPkgInfo">* Local registry package's name is gray colored.</p>
</div>
</div>
</div>
<div ng-show="bundleOrder.length > 1"
@ -34,8 +51,8 @@ limitations under the License.
</div>
</div>
<span class="saveLink"
ng-show="bundleOrderChanged"
ng-click="saveBundleOrder()">
ng-show="bundleOrderChanged"
ng-click="saveBundleOrder()">
save
</span>
</div>
@ -44,28 +61,46 @@ limitations under the License.
</div>
<div class="box width-full heliumPackageContainer">
<div class="row"
style="padding-bottom: 15px"
ng-if="getPackageSize(defaultPackages, pkgListByType) === 0">
<div class="col-md-12 gray40-message">
<em>Currently there is no available package to be listed</em>
</div>
</div>
<div class="row heliumPackageList"
ng-repeat="(pkgName, pkgSearchResult) in defaultPackages">
ng-repeat="pkgSearchResult in defaultPackages | toArray:false | orderBy: 'pkg.published':true"
ng-show="$parent.pkgListByType === pkgSearchResult.pkg.type">
<div class="col-md-12">
<div class="heliumPackageHead">
<div class="heliumPackageIcon"
ng-if="pkgSearchResult.pkg.type !== 'INTERPRETER'"
ng-bind-html=pkgSearchResult.pkg.icon></div>
<div class="heliumPackageIcon"
ng-if="pkgSearchResult.pkg.type === 'INTERPRETER'"
ng-bind-html=intpDefaultIcon></div>
<div class="heliumPackageName">
<span ng-if="hasNpmLink(pkgSearchResult)">
<a target="_blank" href="https://www.npmjs.com/package/{{pkgName}}">{{pkgName}}</a>
<a target="_blank" href="https://www.npmjs.com/package/{{pkgSearchResult.pkg.name}}">{{pkgSearchResult.pkg.name}}</a>
</span>
<span ng-if="!hasNpmLink(pkgSearchResult)" ng-class="{'heliumLocalPackage': isLocalPackage(pkgSearchResult)}">
{{pkgName}}
<span ng-if="!hasNpmLink(pkgSearchResult) && !hasMavenLink(pkgSearchResult)" ng-class="{'heliumLocalPackage': isLocalPackage(pkgSearchResult)}">
{{pkgSearchResult.pkg.name}}
</span>
<span ng-if="hasMavenLink(pkgSearchResult)">
<a target="_blank"
href="http://search.maven.org/#search%7Cga%7C1%7Ca%3A%22{{pkgSearchResult.pkg.artifact.split('@')[0]}}%22%20AND%20v%3A%22{{pkgSearchResult.pkg.artifact.split('@')[1]}}%22">
{{pkgSearchResult.pkg.name}}
</a>
</span>
<span class="heliumType">{{pkgSearchResult.pkg.type}}</span>
</div>
<div ng-show="!pkgSearchResult.enabled"
ng-click="enable(pkgName, pkgSearchResult.pkg.artifact)"
ng-click="enable(pkgSearchResult.pkg.name, pkgSearchResult.pkg.artifact, pkgSearchResult.pkg.type, pkgSearchResult.pkg.groupId)"
class="btn btn-success btn-xs"
style="float:right">Enable</div>
<div ng-show="pkgSearchResult.enabled"
ng-click="disable(pkgName)"
ng-click="disable(pkgSearchResult.pkg.name)"
ng-if="pkgSearchResult.pkg.type !== 'INTERPRETER'"
class="btn btn-info btn-xs"
style="float:right">Disable</div>
<div ng-show="configExists(pkgSearchResult)"
@ -76,20 +111,27 @@ limitations under the License.
</div>
<div ng-class="{heliumPackageDisabledArtifact: !pkgSearchResult.enabled, heliumPackageEnabledArtifact: pkgSearchResult.enabled}">
{{pkgSearchResult.pkg.artifact}}
<span ng-show="pkgSearchResults[pkgName].length > 0"
ng-click="toggleVersions(pkgName)">
<span ng-show="pkgSearchResults[pkgSearchResult.pkg.name].length > 0"
ng-click="toggleVersions(pkgSearchResult.pkg.name)">
versions
</span>
</div>
<ul class="heliumPackageVersions"
ng-show="showVersions[pkgName]">
ng-show="showVersions[pkgSearchResult.pkg.name]">
<li class="heliumPackageDisabledArtifact"
ng-repeat="pkgSearchResult in pkgSearchResults[pkgName]">
ng-repeat="pkgSearchResult in pkgSearchResults[pkgSearchResult.pkg.name]">
{{pkgSearchResult.pkg.artifact}} -
<span ng-click="enable(pkgName, pkgSearchResult.pkg.artifact)"
<span ng-click="enable(pkgSearchResult.pkg.name, pkgSearchResult.pkg.artifact, pkgSearchResult.pkg.type, pkgSearchResult.pkg.groupId)"
ng-if="pkgSearchResult.pkg.type !== 'INTERPRETER'"
style="margin-left:3px;cursor:pointer;text-decoration: underline;color:#3071a9">
enable
</span>
<a target="_blank"
ng-if="pkgSearchResult.pkg.type === 'INTERPRETER'"
style="margin-left:3px;cursor:pointer;text-decoration: underline;color:#3071a9"
href="http://search.maven.org/#search%7Cga%7C1%7Ca%3A%22{{pkgSearchResult.pkg.artifact.split('@')[0]}}%22%20AND%20v%3A%22{{pkgSearchResult.pkg.artifact.split('@')[1]}}%22">
see more
</a>
</li>
</ul>
<div class="heliumPackageDescription">

BIN
zeppelin-web/src/assets/images/maven_default_icon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 4.1 KiB

2
zeppelin-web/src/components/helium/helium-type.js
View file

@ -15,4 +15,6 @@
export const HeliumType = {
VISUALIZATION: 'VISUALIZATION',
SPELL: 'SPELL',
INTERPRETER: 'INTERPRETER',
APPLICATION: 'APPLICATION',
}

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
View file

@ -34,7 +34,7 @@ import java.net.URL;
import java.util.*;
/**
* Load helium visualization
* Load helium visualization & spell
*/
public class HeliumBundleFactory {
Logger logger = LoggerFactory.getLogger(HeliumBundleFactory.class);