Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'apache-github/master' into ZEPPELIN-1619-rebased

Browse source
This commit is contained in:
Lee moon soo 2017-01-10 11:57:48 -08:00
commit ecd925b85b
40 changed files with 356 additions and 202 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -47,6 +47,7 @@ zeppelin-web/src/fonts/google-fonts.css
zeppelin-web/.sass-cache
zeppelin-web/npm-debug.log
zeppelin-web/bower_components
zeppelin-web/yarn.lock
**nbproject/
**node/

3
conf/zeppelin-env.cmd.template
View file

@ -31,6 +31,9 @@ REM set ZEPPELIN_NOTEBOOK_HOMESCREEN REM Id of notebook to be displayed in home
REM set ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE REM hide homescreen notebook from list when this value set to "true". default "false"
REM set ZEPPELIN_NOTEBOOK_S3_BUCKET REM Bucket where notebook saved
REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
REM set ZEPPELIN_NOTEBOOK_S3_ENDPOINT REM Endpoint of the bucket
REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID REM AWS KMS key ID
REM set ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION REM AWS KMS key region
REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default.
REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0.
REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading

2
conf/zeppelin-env.sh.template
View file

@ -33,6 +33,8 @@
# export ZEPPELIN_NOTEBOOK_S3_BUCKET # Bucket where notebook saved
# export ZEPPELIN_NOTEBOOK_S3_ENDPOINT # Endpoint of the bucket
# export ZEPPELIN_NOTEBOOK_S3_USER # User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json
# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID # AWS KMS key ID
# export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION # AWS KMS key region
# export ZEPPELIN_IDENT_STRING # A string representing this instance of zeppelin. $USER by default.
# export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0.
# export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading

10
conf/zeppelin-site.xml.template
View file

@ -108,6 +108,16 @@
</property>
-->
<!-- provide region of your KMS key -->
<!-- See http://docs.aws.amazon.com/general/latest/gr/rande.html#kms_region for region codes names -->
<!--
<property>
<name>zeppelin.notebook.s3.kmsKeyRegion</name>
<value>us-east-1</value>
<description>AWS KMS key region in your AWS account</description>
</property>
-->
<!-- Use a custom encryption materials provider to encrypt data -->
<!-- No configuration is given to the provider, so you must use system properties or another means to configure -->
<!-- See https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/EncryptionMaterialsProvider.html -->

19
docs/storage/storage.md
View file

@ -130,6 +130,23 @@ Or using the following setting in **zeppelin-site.xml**:
</property>
```
In order to set custom KMS key region, set the following environment variable in the file **zeppelin-env.sh**:
```
export ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION = kms-key-region
```
Or using the following setting in **zeppelin-site.xml**:
```
<property>
<name>zeppelin.notebook.s3.kmsKeyRegion</name>
<value>target-region</value>
<description>AWS KMS key region in your AWS account</description>
</property>
```
Format of `target-region` is described in more details [here](http://docs.aws.amazon.com/general/latest/gr/rande.html#kms_region) in second `Region` column (e.g. `us-east-1`).
#### Custom Encryption Materials Provider class
You may use a custom [``EncryptionMaterialsProvider``](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/EncryptionMaterialsProvider.html) class as long as it is available in the classpath and able to initialize itself from system properties or another mechanism. To use this, set the following environment variable in the file **zeppelin-env.sh**:
@ -238,4 +255,4 @@ export ZEPPELINHUB_API_TOKEN = ZeppelinHub token
export ZEPPELINHUB_API_ADDRESS = address of ZeppelinHub service (e.g. https://www.zeppelinhub.com)
```
You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).
You can get more information on generating `token` and using authentication on the corresponding [help page](http://help.zeppelinhub.com/zeppelin_integration/#add-a-new-zeppelin-instance-and-generate-a-token).

1
spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java
View file

@ -230,5 +230,4 @@ public class SparkRInterpreter extends Interpreter {
return false;
}
}
}

14
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinR.java
View file

@ -57,7 +57,6 @@ public class ZeppelinR implements ExecuteResultHandler {
boolean rScriptInitialized = false;
Integer rScriptInitializeNotifier = new Integer(0);
/**
* Request to R repl
*/
@ -103,8 +102,6 @@ public class ZeppelinR implements ExecuteResultHandler {
boolean rResponseError = false;
Integer rResponseNotifier = new Integer(0);
/**
* Create ZeppelinR instance
* @param rCmdPath R repl commandline path
@ -216,7 +213,6 @@ public class ZeppelinR implements ExecuteResultHandler {
}
}
/**
* Send request to r repl and return response
* @return responseValue
@ -257,7 +253,6 @@ public class ZeppelinR implements ExecuteResultHandler {
}
}
/**
* Wait until src/main/resources/R/zeppelin_sparkr.R is initialized
* and call onScriptInitialized()
@ -286,14 +281,11 @@ public class ZeppelinR implements ExecuteResultHandler {
e.printStackTrace();
}
if (rScriptInitialized == false) {
throw new InterpreterException("sparkr is not responding " + errorMessage);
}
}
/**
* invoked by src/main/resources/R/zeppelin_sparkr.R
* @return
@ -337,7 +329,6 @@ public class ZeppelinR implements ExecuteResultHandler {
}
}
/**
* Create R script in tmp dir
*/
@ -381,7 +372,6 @@ public class ZeppelinR implements ExecuteResultHandler {
return zeppelinR.get(hashcode);
}
/**
* Pass InterpreterOutput to capture the repl output
* @param out
@ -390,8 +380,6 @@ public class ZeppelinR implements ExecuteResultHandler {
outputStream.setInterpreterOutput(out);
}
@Override
public void onProcessComplete(int i) {
logger.info("process complete {}", i);
@ -403,6 +391,4 @@ public class ZeppelinR implements ExecuteResultHandler {
logger.error(e.getMessage(), e);
rScriptRunning = false;
}
}

4
spark/src/main/scala/org/apache/zeppelin/spark/ZeppelinRDisplay.scala
View file

@ -88,7 +88,6 @@ object ZeppelinRDisplay {
}
private def htmlDisplay(body: Element, imageWidth: String): RDisplay = {
var div = new String()
for (element <- body.children) {
@ -101,7 +100,6 @@ object ZeppelinRDisplay {
val r = (pattern findFirstIn eHtml).getOrElse("")
div = div + eOuterHtml.replace(r, "")
}
val content = div
@ -115,7 +113,5 @@ object ZeppelinRDisplay {
}
RDisplay(body.html, HTML, SUCCESS)
}
}

22
zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
View file

@ -208,6 +208,28 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
}
}
/**
* Close all interpreter instances in this group
*/
public void shutdown() {
LOGGER.info("Close interpreter group " + getId());
// make sure remote interpreter process terminates
if (remoteInterpreterProcess != null) {
while (remoteInterpreterProcess.referenceCount() > 0) {
remoteInterpreterProcess.dereference();
}
remoteInterpreterProcess = null;
}
allInterpreterGroups.remove(id);
List<Interpreter> intpToClose = new LinkedList<>();
for (List<Interpreter> intpGroupForSession : this.values()) {
intpToClose.addAll(intpGroupForSession);
}
close(intpToClose);
}
public void setResourcePool(ResourcePool resourcePool) {
this.resourcePool = resourcePool;
}

3
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -187,8 +187,9 @@ public class ZeppelinServer extends Application {
LOG.info("Shutting down Zeppelin Server ... ");
try {
jettyWebServer.stop();
notebook.getInterpreterFactory().close();
notebook.getInterpreterFactory().shutdown();
notebook.close();
Thread.sleep(3000);
} catch (Exception e) {
LOG.error("Error while stopping servlet container", e);
}

88
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -36,6 +36,7 @@ import javax.servlet.http.HttpServletRequest;
import com.google.common.base.Strings;
import com.google.common.collect.Sets;
import com.google.gson.*;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@ -85,8 +86,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Queues;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
/**
@ -263,6 +262,9 @@ public class NotebookServer extends WebSocketServlet
case RUN_PARAGRAPH:
runParagraph(conn, userAndRoles, notebook, messagereceived);
break;
case RUN_ALL_PARAGRAPHS:
runAllParagraphs(conn, userAndRoles, notebook, messagereceived);
break;
case CANCEL_PARAGRAPH:
cancelParagraph(conn, userAndRoles, notebook, messagereceived);
break;
@ -1534,8 +1536,46 @@ public class NotebookServer extends WebSocketServlet
p.abort();
}
private void runParagraph(NotebookSocket conn, HashSet<String> userAndRoles, Notebook notebook,
private void runAllParagraphs(NotebookSocket conn, HashSet<String> userAndRoles,
Notebook notebook,
Message fromMessage) throws IOException {
final String noteId = (String) fromMessage.get("noteId");
if (StringUtils.isBlank(noteId)) {
return;
}
Note note = notebook.getNote(noteId);
NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
permissionError(conn, "run all paragraphs", fromMessage.principal, userAndRoles,
notebookAuthorization.getOwners(noteId));
return;
}
List<Map<String, Object>> paragraphs =
gson.fromJson(String.valueOf(fromMessage.data.get("paragraphs")),
new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> raw : paragraphs) {
String paragraphId = (String) raw.get("id");
if (paragraphId == null) {
continue;
}
String text = (String) raw.get("paragraph");
String title = (String) raw.get("title");
Map<String, Object> params = (Map<String, Object>) raw.get("params");
Map<String, Object> config = (Map<String, Object>) raw.get("config");
Paragraph p = setParagraphUsingMessage(note, fromMessage,
paragraphId, text, title, params, config);
persistAndExecuteSingleParagraph(conn, note, p);
}
}
private void runParagraph(NotebookSocket conn, HashSet<String> userAndRoles, Notebook notebook,
Message fromMessage) throws IOException {
final String paragraphId = (String) fromMessage.get("id");
if (paragraphId == null) {
return;
@ -1550,30 +1590,29 @@ public class NotebookServer extends WebSocketServlet
return;
}
Paragraph p = note.getParagraph(paragraphId);
String text = (String) fromMessage.get("paragraph");
p.setText(text);
p.setTitle((String) fromMessage.get("title"));
AuthenticationInfo subject =
new AuthenticationInfo(fromMessage.principal, fromMessage.ticket);
p.setAuthenticationInfo(subject);
String title = (String) fromMessage.get("title");
Map<String, Object> params = (Map<String, Object>) fromMessage.get("params");
p.settings.setParams(params);
Map<String, Object> config = (Map<String, Object>) fromMessage.get("config");
p.setConfig(config);
Paragraph p = setParagraphUsingMessage(note, fromMessage, paragraphId,
text, title, params, config);
persistAndExecuteSingleParagraph(conn, note, p);
}
private void persistAndExecuteSingleParagraph(NotebookSocket conn,
Note note, Paragraph p) throws IOException {
// if it's the last paragraph and empty, let's add a new one
boolean isTheLastParagraph = note.isLastParagraph(p.getId());
if (!(text.trim().equals(p.getMagic()) ||
Strings.isNullOrEmpty(text)) &&
if (!(p.getText().trim().equals(p.getMagic()) ||
Strings.isNullOrEmpty(p.getText())) &&
isTheLastParagraph) {
Paragraph newPara = note.addParagraph(subject);
Paragraph newPara = note.addParagraph(p.getAuthenticationInfo());
broadcastNewParagraph(note, newPara);
}
try {
note.persist(subject);
note.persist(p.getAuthenticationInfo());
} catch (FileSystemException ex) {
LOG.error("Exception from run", ex);
conn.send(serializeMessage(new Message(OP.ERROR_INFO).put("info",
@ -1584,7 +1623,7 @@ public class NotebookServer extends WebSocketServlet
}
try {
note.run(paragraphId);
note.run(p.getId());
} catch (Exception ex) {
LOG.error("Exception from run", ex);
if (p != null) {
@ -1595,6 +1634,21 @@ public class NotebookServer extends WebSocketServlet
}
}
private Paragraph setParagraphUsingMessage(Note note, Message fromMessage, String paragraphId,
String text, String title, Map<String, Object> params,
Map<String, Object> config) {
Paragraph p = note.getParagraph(paragraphId);
p.setText(text);
p.setTitle(title);
AuthenticationInfo subject =
new AuthenticationInfo(fromMessage.principal, fromMessage.ticket);
p.setAuthenticationInfo(subject);
p.settings.setParams(params);
p.setConfig(config);
return p;
}
private void sendAllConfigurations(NotebookSocket conn, HashSet<String> userAndRoles,
Notebook notebook) throws IOException {
ZeppelinConfiguration conf = notebook.getConf();

2
zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java
View file

@ -241,7 +241,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT {
driver.findElement(By.xpath(getParagraphXPath(1) + "//span[@class='icon-control-play shortcut-icon']")).isDisplayed(), CoreMatchers.equalTo(false)
);
driver.findElement(By.xpath(".//*[@id='main']//button[@ng-click='runNote()']")).sendKeys(Keys.ENTER);
driver.findElement(By.xpath(".//*[@id='main']//button[contains(@ng-click, 'runAllParagraphs')]")).sendKeys(Keys.ENTER);
ZeppelinITUtils.sleep(1000, true);
driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'Run all paragraphs?')]" +
"//div[@class='modal-footer']//button[contains(.,'OK')]")).click();

15
zeppelin-web/README.md
View file

@ -21,17 +21,18 @@ All build commands are described in [package.json](./package.json)
```sh
# install required depepdencies and bower packages (only once)
$ npm install
$ npm install -g yarn
$ yarn install
# build zeppelin-web for production
$ npm run build
$ yarn run build
# run frontend application only in dev mode (localhost:9000)
# you need to run zeppelin backend instance also
$ npm run dev
$ yarn run dev
# execute tests
$ npm run test
$ yarn run test
```
## Troubleshooting
@ -50,12 +51,12 @@ Try to add to the `.bowerrc` file the following content:
"https-proxy" : "http://<host>:<port>"
```
also try to add proxy info to npm install command:
also try to add proxy info to yarn install command:
```xml
<execution>
<id>npm install</id>
<id>yarn install</id>
<goals>
<goal>npm</goal>
<goal>yarn</goal>
</goals>
<configuration>
<arguments>--proxy=http://<host>:<port> --https-proxy=http://<host>:<port></arguments>

2
zeppelin-web/package.json
View file

@ -10,7 +10,7 @@
"postinstall": "bower install --silent && grunt googlefonts",
"prebuild": "npm-run-all clean",
"build": "grunt pre-webpack-dist && webpack && grunt post-webpack-dist",
"predev": "npm-run-all clean && grunt pre-webpack-dev",
"predev": "grunt pre-webpack-dev",
"dev:server": "webpack-dev-server --hot",
"visdev:server": "HELIUM_VIS_DEV=true webpack-dev-server --hot",
"dev:watch": "grunt watch-webpack-dev",

23
zeppelin-web/pom.xml
View file

@ -39,7 +39,7 @@
<properties>
<node.version>v6.9.1</node.version>
<npm.version>3.10.8</npm.version>
<yarn.version>v0.18.1</yarn.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!--plugin versions-->
@ -94,6 +94,7 @@
<exclude>bower.json</exclude>
<exclude>**/package.json</exclude>
<exclude>**/.npmignore</exclude>
<exclude>yarn.lock</exclude>
<exclude>*.md</exclude>
</excludes>
</configuration>
@ -106,30 +107,30 @@
<executions>
<execution>
<id>install node and npm</id>
<id>install node and yarn</id>
<goals>
<goal>install-node-and-npm</goal>
<goal>install-node-and-yarn</goal>
</goals>
<configuration>
<nodeVersion>${node.version}</nodeVersion>
<npmVersion>${npm.version}</npmVersion>
<yarnVersion>${yarn.version}</yarnVersion>
</configuration>
</execution>
<execution>
<id>npm install</id>
<id>yarn install</id>
<goals>
<goal>npm</goal>
<goal>yarn</goal>
</goals>
<configuration>
<arguments>install</arguments>
<arguments>install --no-lockfile</arguments>
</configuration>
</execution>
<execution>
<id>npm build</id>
<id>yarn build</id>
<goals>
<goal>npm</goal>
<goal>yarn</goal>
</goals>
<configuration>
<arguments>run build</arguments>
@ -137,9 +138,9 @@
</execution>
<execution>
<id>npm test</id>
<id>yarn test</id>
<goals>
<goal>npm</goal>
<goal>yarn</goal>
</goals>
<phase>test</phase>
<configuration>

1
zeppelin-web/src/app/handsontable/handsonHelper.js
View file

@ -20,6 +20,7 @@ export default class HandsonHelper {
this.columns = columns || [];
this.rows = rows || [];
this.comment = comment || '';
this._numericValidator = this._numericValidator.bind(this);
};
getHandsonTableConfig(columns, columnNames, resultRows) {

4
zeppelin-web/src/app/home/home.css
View file

@ -381,13 +381,13 @@ a.navbar-brand:hover {
}
.zeppelin {
background: url('/assets/images/zepLogo.png') no-repeat right;
background: url('../assets/images/zepLogo.png') no-repeat right;
height: 380px;
opacity: 0.2;
}
.zeppelin2 {
background: url('/assets/images/zepLogo.png') no-repeat right;
background: url('../assets/images/zepLogo.png') no-repeat right;
background-position-y: 12px;
height: 380px;
opacity: 0.2;

2
zeppelin-web/src/app/home/home.html
View file

@ -133,7 +133,7 @@ limitations under the License.
<div>
<h5><a href="" data-toggle="modal" data-target="#noteImportModal" style="text-decoration: none;">
<i style="font-size: 15px;" class="fa fa-upload"></i> Import note</a></h5>
<h5><a href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;">
<h5 ng-controller="NotenameCtrl as notenamectrl"><a href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;" ng-click="notenamectrl.getInterpreterSettings()">
<i style="font-size: 15px;" class="icon-notebook"></i> Create new note</a></h5>
<ul id="notebook-names">
<li class="filter-names" ng-include="'components/filterNoteNames/filter-note-names.html'"></li>

2
zeppelin-web/src/app/notebook/notebook-actionBar.html
View file

@ -24,7 +24,7 @@ limitations under the License.
<span class="labelBtn btn-group">
<button type="button"
class="btn btn-default btn-xs"
ng-click="runNote()"
ng-click="runAllParagraphs(note.id)"
ng-class="{'disabled':isNoteRunning()}"
tooltip-placement="bottom" tooltip="Run all paragraphs"
ng-disabled="revisionView">

21
zeppelin-web/src/app/notebook/notebook.controller.js
View file

@ -286,16 +286,23 @@
}
};
$scope.runNote = function() {
$scope.runAllParagraphs = function(noteId) {
BootstrapDialog.confirm({
closable: true,
title: '',
message: 'Run all paragraphs?',
callback: function(result) {
if (result) {
_.forEach($scope.note.paragraphs, function(n, key) {
angular.element('#' + n.id + '_paragraphColumn_main').scope().runParagraph(n.text);
const paragraphs = $scope.note.paragraphs.map(p => {
return {
id: p.id,
title: p.title,
paragraph: p.text,
config: p.config,
params: p.settings.params
};
});
websocketMsgSrv.runAllParagraphs(noteId, paragraphs);
}
}
});
@ -582,6 +589,14 @@
}
websocketMsgSrv.saveInterpreterBindings($scope.note.id, selectedSettingIds);
console.log('Interpreter bindings %o saved', selectedSettingIds);
_.forEach($scope.note.paragraphs, function(n, key) {
var regExp = /^\s*%/g;
if (n.text && !regExp.exec(n.text)) {
$scope.$broadcast('saveInterpreterBindings', n.id);
}
});
$scope.showSetting = false;
};

2
zeppelin-web/src/app/notebook/paragraph/paragraph-progressBar.html
View file

@ -13,7 +13,7 @@ limitations under the License.
-->
<div id="{{paragraph.id}}_runControl" class="runControl">
<div id="{{paragraph.id}}_progress" class="progress" ng-show="paragraph.status=='RUNNING'">
<div id="{{paragraph.id}}_progress" class="progress" ng-if="paragraph.status=='RUNNING'">
<div ng-if="getProgress()>0 && getProgress()<100 && paragraph.status=='RUNNING'"
class="progress-bar" role="progressbar" style="width:{{getProgress()}}%;"></div>
<div ng-if="(getProgress()<=0 || getProgress()>=100) && (paragraph.status=='RUNNING' )"

14
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -43,7 +43,10 @@
$scope.editor = null;
var editorSetting = {};
// flag that is used to set editor setting on paste percent sign
var pastePercentSign = false;
// flag that is used to set editor setting on save interpreter bindings
var setInterpreterBindings = false;
var paragraphScope = $rootScope.$new(true, $rootScope);
// to keep backward compatibility
@ -661,7 +664,8 @@
if ((typeof pos === 'undefined') || (pos.row === 0 && pos.column < 30) ||
(pos.row === 1 && pos.column === 0) || pastePercentSign) {
// If paragraph loading, use config value if exists
if ((typeof pos === 'undefined') && $scope.paragraph.config.editorMode) {
if ((typeof pos === 'undefined') && $scope.paragraph.config.editorMode &&
!setInterpreterBindings) {
session.setMode($scope.paragraph.config.editorMode);
} else {
var magic = getInterpreterName(paragraphText);
@ -676,6 +680,7 @@
}
}
pastePercentSign = false;
setInterpreterBindings = false;
};
var getInterpreterName = function(paragraphText) {
@ -1154,6 +1159,13 @@
}
});
$scope.$on('saveInterpreterBindings', function(event, paragraphId) {
if ($scope.paragraph.id === paragraphId) {
setInterpreterBindings = true;
setParagraphMode($scope.editor.getSession(), $scope.editor.getSession().getValue());
}
});
$scope.$on('doubleClickParagraph', function(event, paragraphId) {
if ($scope.paragraph.id === paragraphId && $scope.paragraph.config.editorHide &&
$scope.paragraph.config.editorSetting.editOnDblClick && $scope.revisionView !== true) {

4
zeppelin-web/src/app/notebook/paragraph/paragraph.css
View file

@ -72,6 +72,10 @@ table.dataTable.table-condensed .sorting_desc:after {
font-size: 12px !important;
}
.plainTextContent{
white-space:pre-wrap; /** to preserve white-space and newlines of result */
}
.graphContainer {
position: relative;
margin-bottom: 5px;

3
zeppelin-web/src/app/notebook/paragraph/paragraph.html
View file

@ -56,8 +56,7 @@ limitations under the License.
ng-controller="ResultCtrl"
ng-repeat="result in paragraph.results.msg track by $index"
ng-init="init(result, paragraph.config.results[$index], paragraph, $index)"
ng-include src="'app/notebook/paragraph/result/result.html'"
>
ng-include src="'app/notebook/paragraph/result/result.html'">
</div>
<div id="{{paragraph.id}}_error"
class="error text"

47
zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html
View file

@ -14,31 +14,33 @@ limitations under the License.
<div id="{{id}}_switch"
ng-if="(type == 'TABLE' || apps.length > 0 || suggestion.available && suggestion.available.length > 0) && !asIframe && !viewOnly"
class="btn-group result-chart-selector"
style="margin-bottom: 10px;">
class="result-chart-selector">
<button type="button" class="btn btn-default btn-sm"
ng-if="type == 'TABLE'"
ng-repeat="viz in builtInTableDataVisualizationList track by $index"
ng-class="{'active' : viz.id == graphMode && !config.helium.activeApp}"
ng-click="switchViz(viz.id)"
ng-bind-html="viz.icon">
</button>
<div ng-if="type == 'TABLE'" class="btn-group">
<button type="button" class="btn btn-default btn-sm"
ng-repeat="viz in builtInTableDataVisualizationList track by $index"
ng-class="{'active' : viz.id == graphMode && !config.helium.activeApp}"
ng-click="switchViz(viz.id)"
ng-bind-html="viz.icon">
</button>
</div>
<button type="button"
ng-if="type != 'TABLE'"
ng-click="switchApp()"
ng-class="{'active' : !config.helium.activeApp}"
class="btn btn-default btn-sm"><i class="fa fa-terminal"></i>
</button>
<div class="btn-group">
<button type="button"
ng-if="type != 'TABLE'"
ng-click="switchApp()"
ng-class="{'active' : !config.helium.activeApp}"
class="btn btn-default btn-sm"><i class="fa fa-terminal"></i>
</button>
<button type="button"
class="btn btn-default btn-sm"
ng-repeat="app in apps"
ng-click="switchApp(app.id)"
ng-class="{'active' : app.id == config.helium.activeApp}"
ng-bind-html="app.pkg.icon">
</button>
<button type="button"
class="btn btn-default btn-sm"
ng-repeat="app in apps"
ng-click="switchApp(app.id)"
ng-class="{'active' : app.id == config.helium.activeApp}"
ng-bind-html="app.pkg.icon">
</button>
</div>
</div>
<div id="{{paragraph.id}}_helium"
@ -47,7 +49,6 @@ limitations under the License.
style="margin-bottom: 10px;">
<button type="button"
class="btn btn-default btn-sm dropdown-toggle"
ng-if="suggestion.available && suggestion.available.length > 0"
data-toggle="dropdown"
style="font-weight:bold; background-color:#ffdf96; border: 1px solid #FED233">
He

62
zeppelin-web/src/app/notebook/paragraph/result/result-results.html
View file

@ -1,62 +0,0 @@
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div
id="p{{id}}_resize"
ng-if="!config.helium.activeApp"
style="padding-bottom: 5px;"
resize='{"allowresize": "{{!asIframe && !viewOnly}}", "graphType": "{{type}}"}'
resizable on-resize="resize(width, height);">
<div ng-include src="'app/notebook/paragraph/result/result-graph.html'"></div>
<div id="{{id}}_comment"
class="text"
ng-if="type == 'TABLE' && tableDataComment"
ng-bind-html="tableDataComment">
</div>
<div id="p{{id}}_text"
ng-if="type == 'TEXT'">
<div class="fa fa-level-down scroll-paragraph-down"
ng-show="showScrollDownIcon(id)"
ng-click="scrollParagraphDown(id)"
tooltip="Follow Output"></div>
<div id="p{{id}}_text"
style="max-height: {{config.graph.height}}px; overflow: auto"
class="text"></div>
<div class="fa fa-chevron-up scroll-paragraph-up"
ng-show="showScrollUpIcon(id)"
ng-click="scrollParagraphUp(id)"
tooltip="Scroll Top"></div>
</div>
<div id="p{{id}}_html"
class="resultContained"
ng-if="type == 'HTML'">
</div>
<div id="p{{id}}_angular"
class="resultContained"
ng-if="type == 'ANGULAR'">
</div>
<img id="{{id}}_img"
ng-if="type == 'IMG'"
ng-src="{{getBase64ImageSrc(result.data)}}" />
</div>
<div ng-repeat="app in apps">
<div id="p{{app.id}}"
ng-show="config.helium.activeApp == app.id">
</div>
</div>

14
zeppelin-web/src/app/notebook/paragraph/result/result.controller.js
View file

@ -316,10 +316,14 @@ import ScatterchartVisualization from '../../../visualization/builtins/visualiza
$timeout(retryRenderer);
};
var getTextEl = function (paragraphId) {
return angular.element('#p' + $scope.id + '_text');
}
var textRendererInitialized = false;
var renderText = function() {
var retryRenderer = function() {
var textEl = angular.element('#p' + $scope.id + '_text');
var textEl = getTextEl($scope.id);
if (textEl.length) {
// clear all lines before render
clearTextOutput();
@ -331,7 +335,7 @@ import ScatterchartVisualization from '../../../visualization/builtins/visualiza
flushAppendQueue();
}
angular.element('#p' + $scope.id + '_text').bind('mousewheel', function(e) {
getTextEl($scope.id).bind('mousewheel', function(e) {
$scope.keepScrollDown = false;
});
} else {
@ -342,7 +346,7 @@ import ScatterchartVisualization from '../../../visualization/builtins/visualiza
};
var clearTextOutput = function() {
var textEl = angular.element('#p' + $scope.id + '_text');
var textEl = getTextEl($scope.id);
if (textEl.length) {
textEl.children().remove();
}
@ -361,7 +365,7 @@ import ScatterchartVisualization from '../../../visualization/builtins/visualiza
textAppendQueueBeforeInitialize.push(msg);
} else {
flushAppendQueue();
var textEl = angular.element('#p' + $scope.id + '_text');
var textEl = getTextEl($scope.id);
if (textEl.length) {
var lines = msg.split('\n');
for (var i = 0; i < lines.length; i++) {
@ -369,7 +373,7 @@ import ScatterchartVisualization from '../../../visualization/builtins/visualiza
}
}
if ($scope.keepScrollDown) {
var doc = angular.element('#p' + $scope.id + '_text');
var doc = getTextEl($scope.id);
doc[0].scrollTop = doc[0].scrollHeight;
}
}

7
zeppelin-web/src/app/notebook/paragraph/result/result.css
View file

@ -12,6 +12,13 @@
* limitations under the License.
*/
.result-chart-selector {
margin-bottom: 10px;
position: relative;
display: inline-block;
vertical-align: middle;
}
.result-chart-selector button {
width: 35px;
height: 30px;

58
zeppelin-web/src/app/notebook/paragraph/result/result.html
View file

@ -21,38 +21,38 @@ limitations under the License.
resize='{"allowresize": "{{!asIframe && !viewOnly}}", "graphType": "{{type}}"}'
resizable on-resize="resize(width, height);">
<!-- graph setting -->
<div class="option lightBold" style="overflow: visible;"
ng-show="type=='TABLE' && graphMode!='table'
&& config.graph.optionOpen && !asIframe && !viewOnly">
<div ng-repeat="viz in builtInTableDataVisualizationList track by $index"
id="trsetting{{id}}_{{viz.id}}"
ng-show="graphMode == viz.id"></div>
<div ng-repeat="viz in builtInTableDataVisualizationList track by $index"
id="vizsetting{{id}}_{{viz.id}}"
ng-show="graphMode == viz.id"></div>
</div>
<div ng-if="type=='TABLE'">
<!-- graph setting -->
<div class="option lightBold" style="overflow: visible;"
ng-if="graphMode!='table'
&& config.graph.optionOpen && !asIframe && !viewOnly">
<div ng-repeat="viz in builtInTableDataVisualizationList track by $index"
id="trsetting{{id}}_{{viz.id}}"
ng-show="graphMode == viz.id"></div>
<div ng-repeat="viz in builtInTableDataVisualizationList track by $index"
id="vizsetting{{id}}_{{viz.id}}"
ng-show="graphMode == viz.id"></div>
</div>
<!-- graph -->
<div id="p{{id}}_graph"
class="graphContainer"
ng-class="{'noOverflow': graphMode=='table'}"
ng-show="type =='TABLE'"
>
<div ng-repeat="viz in builtInTableDataVisualizationList track by $index"
id="p{{id}}_{{viz.id}}"
ng-show="graphMode == viz.id">
<!-- graph -->
<div id="p{{id}}_graph"
class="graphContainer"
ng-class="{'noOverflow': graphMode=='table'}"
>
<div ng-repeat="viz in builtInTableDataVisualizationList track by $index"
id="p{{id}}_{{viz.id}}"
ng-show="graphMode == viz.id">
</div>
</div>
<div id="{{id}}_comment"
class="text"
ng-if="tableDataComment"
ng-bind-html="tableDataComment">
</div>
</div>
<div id="{{id}}_comment"
class="text"
ng-if="type == 'TABLE' && tableDataComment"
ng-bind-html="tableDataComment">
</div>
<div id="p{{id}}_text"
ng-if="type == 'TEXT'"
<div ng-if="type == 'TEXT'"
class="plainTextContainer">
<div class="fa fa-level-down scroll-paragraph-down"
ng-show="showScrollDownIcon()"
@ -60,7 +60,7 @@ limitations under the License.
tooltip="Follow Output"></div>
<div id="p{{id}}_text"
style="max-height: {{config.graph.height}}px; overflow: auto"
class="text"></div>
class="text plainTextContent"></div>
<div class="fa fa-chevron-up scroll-paragraph-up"
ng-show="showScrollUpIcon()"
ng-click="scrollParagraphUp()"

2
zeppelin-web/src/components/navbar/navbar.html
View file

@ -28,7 +28,7 @@ limitations under the License.
<li class="dropdown notebook-list-dropdown" dropdown>
<a href="#" class="dropdown-toggle" data-toggle="dropdown" dropdown-toggle>Notebook <span class="caret"></span></a>
<ul class="dropdown-menu navbar-dropdown-maxHeight" role="menu">
<li><a href="" data-toggle="modal" data-target="#noteNameModal"><i class="fa fa-plus"></i> Create new note</a></li>
<li ng-controller="NotenameCtrl as notenamectrl"><a href="" data-toggle="modal" data-target="#noteNameModal" ng-click="notenamectrl.getInterpreterSettings()"><i class="fa fa-plus"></i> Create new note</a></li>
<li class="divider"></li>
<div id="notebook-list" class="scrollbar-container" ng-if="isDrawNavbarNoteList">
<li class="filter-names" ng-include="'components/filterNoteNames/filter-note-names.html'"></li>

8
zeppelin-web/src/components/noteName-create/notename.controller.js
View file

@ -100,6 +100,7 @@
vm.getInterpreterSettings = function() {
vm.websocketMsgSrv.getInterpreterSettings();
};
$scope.$on('interpreterSettings', function(event, data) {
$scope.interpreterSettings = data.interpreterSettings;
@ -108,13 +109,6 @@
$scope.note.defaultInterpreter = data.interpreterSettings[0];
});
var init = function() {
if (!vm.clone) {
vm.getInterpreterSettings();
}
};
init();
}
})();

10
zeppelin-web/src/components/websocketEvents/websocketMsg.service.js
View file

@ -173,6 +173,16 @@
});
},
runAllParagraphs: function(noteId, paragraphs) {
websocketEvents.sendNewEvent({
op: 'RUN_ALL_PARAGRAPHS',
data: {
noteId: noteId,
paragraphs: JSON.stringify(paragraphs)
}
});
},
removeParagraph: function(paragraphId) {
websocketEvents.sendNewEvent({op: 'PARAGRAPH_REMOVE', data: {id: paragraphId}});
},

9
zeppelin-web/src/index.html
View file

@ -105,7 +105,7 @@ limitations under the License.
<!-- endbuild -->
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
var config = {
extensions: ["tex2jax.js"],
jax: ["input/TeX", "output/HTML-CSS"],
tex2jax: {
@ -115,7 +115,12 @@ limitations under the License.
},
"HTML-CSS": { availableFonts: ["TeX"] },
messageStyle: "none"
});
}
// add root only if it's not dev mode
if (Number(location.port) !== 9000) {
config.root = '.';
}
MathJax.Hub.Config(config);
</script>
<!-- build:js(.) scripts/vendor.js -->

2
zeppelin-web/test/spec/controllers/notebook.js
View file

@ -35,7 +35,7 @@ describe('Controller: NotebookCtrl', function() {
scope.note = noteMock;
});
var functions = ['getCronOptionNameFromValue', 'removeNote', 'runNote', 'saveNote', 'toggleAllEditor',
var functions = ['getCronOptionNameFromValue', 'removeNote', 'runAllParagraphs', 'saveNote', 'toggleAllEditor',
'showAllEditor', 'hideAllEditor', 'toggleAllTable', 'hideAllTable', 'showAllTable', 'isNoteRunning',
'killSaveTimer', 'startSaveTimer', 'setLookAndFeel', 'setCronScheduler', 'setConfig', 'updateNoteName',
'openSetting', 'closeSetting', 'saveSetting', 'toggleSetting'];

2
zeppelin-web/webpack.config.js
View file

@ -98,7 +98,7 @@ module.exports = function makeWebpackConfig () {
// Output path from the view of the page
// Uses webpack-dev-server in development
publicPath: isProd ? '/' : 'http://localhost:9000/',
publicPath: isProd ? '' : 'http://localhost:9000/',
// Filename for entry points
// Only adds hash in build mode

5
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -369,6 +369,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID);
}
public String getS3KMSKeyRegion() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION);
}
public String getS3EncryptionMaterialsProviderClass() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
}
@ -579,6 +583,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),
ZEPPELIN_NOTEBOOK_S3_EMP("zeppelin.notebook.s3.encryptionMaterialsProvider", null),
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),

24
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
View file

@ -1049,6 +1049,30 @@ public class InterpreterFactory implements InterpreterGroupFactory {
}
}
public void shutdown() {
List<Thread> closeThreads = new LinkedList<>();
synchronized (interpreterSettings) {
Collection<InterpreterSetting> intpSettings = interpreterSettings.values();
for (final InterpreterSetting intpSetting : intpSettings) {
Thread t = new Thread() {
public void run() {
intpSetting.shutdownAndRemoveAllInterpreterGroups();
}
};
t.start();
closeThreads.add(t);
}
}
for (Thread t : closeThreads) {
try {
t.join();
} catch (InterruptedException e) {
logger.error("Can't close interpreterGroup", e);
}
}
}
private Interpreter createRepl(String dirName, String className, Properties property)
throws InterpreterException {
logger.info("Create repl {} from {}", className, dirName);

26
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
View file

@ -197,6 +197,32 @@ public class InterpreterSetting {
}
}
void shutdownAndRemoveInterpreterGroup(String interpreterGroupKey) {
String key = getInterpreterProcessKey("", interpreterGroupKey);
List<InterpreterGroup> groupToRemove = new LinkedList<>();
InterpreterGroup groupItem;
for (String intpKey : new HashSet<>(interpreterGroupRef.keySet())) {
if (intpKey.contains(key)) {
interpreterGroupWriteLock.lock();
groupItem = interpreterGroupRef.remove(intpKey);
interpreterGroupWriteLock.unlock();
groupToRemove.add(groupItem);
}
}
for (InterpreterGroup groupToClose : groupToRemove) {
groupToClose.shutdown();
}
}
void shutdownAndRemoveAllInterpreterGroups() {
HashSet<String> groupsToRemove = new HashSet<>(interpreterGroupRef.keySet());
for (String interpreterGroupKey : groupsToRemove) {
shutdownAndRemoveInterpreterGroup(interpreterGroupKey);
}
}
public Object getProperties() {
return properties;
}

17
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -31,6 +31,7 @@ import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.notebook.Note;
@ -48,12 +49,15 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.AmazonS3EncryptionClient;
import com.amazonaws.services.s3.model.CryptoConfiguration;
import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.google.gson.Gson;
@ -91,13 +95,24 @@ public class S3NotebookRepo implements NotebookRepo {
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
CryptoConfiguration cryptoConf = null;
String keyRegion = conf.getS3KMSKeyRegion();
if (StringUtils.isNotBlank(keyRegion)) {
cryptoConf = new CryptoConfiguration();
cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
}
// see if we should be encrypting data in S3
String kmsKeyID = conf.getS3KMSKeyID();
if (kmsKeyID != null) {
// use the AWS KMS to encrypt data
KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
if (cryptoConf != null) {
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cryptoConf);
} else {
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
}
}
else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
// use a custom encryption materials provider class

3
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java
View file

@ -171,7 +171,8 @@ public class Message {
PARAGRAPH_ADDED, // [s-c] paragraph is added
PARAGRAPH_REMOVED, // [s-c] paragraph deleted
PARAGRAPH_MOVED, // [s-c] paragraph moved
NOTE_UPDATED // [s-c] paragraph updated(name, config)
NOTE_UPDATED, // [s-c] paragraph updated(name, config)
RUN_ALL_PARAGRAPHS // [c-s] run all paragraphs
}
public static final Message EMPTY = new Message(null);