Merge with master

This commit is contained in:
Eric Charles 2016-04-01 09:32:32 +02:00
commit ed70820f50
19 changed files with 333 additions and 36 deletions

View file

@ -15,26 +15,21 @@
language: java
addons:
apt:
sources:
r-packages-precise
packages:
r-base
sudo: false
matrix:
include:
# Test all modules
- jdk: "oraclejdk7"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS=""
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark -Psparkr" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.rest.*Test,org.apache.zeppelin.spark* -DfailIfNoTests=false"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
@ -56,18 +51,13 @@ before_install:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
# install R packages
- mkdir -p ~/Rlib
- echo 'R_LIBS=~/Rlib' > ~/.Renviron
- Rscript -e "install.packages('knitr', repos = 'http://cran.us.r-project.org')"
install:
- mvn $BUILD_FLAG $PROFILE -B
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
- ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
- echo -e "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER\nexport ZEPPELIN_R_CMD=Rscript" > conf/zeppelin-env.sh
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
script:
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS
@ -81,6 +71,3 @@ after_failure:
after_script:
- ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER
notifications:
slack:
secure: dtIkPwlf5uTun19p9TtPEAFmrLOMK2COE8TL9m8LXX/N2WzJaKYvAnovMObEV6KEgK2oZ+72Cke7eBI+Hp4FmHZ2B7mQI/PNCfRZthI3cc3zVmMd25yvLH9AlCRa2bC6R885z2copvzaoZtLBkHnPa8bUrUkbmRp40qkDPQpgO4=

View file

@ -129,6 +129,7 @@ Available profiles are
-Pmapr40
-Pmapr41
-Pmapr50
-Pmapr51
```

View file

@ -29,11 +29,12 @@ user3 = password4, role2
#ldapRealm.userDnTemplate = cn={0},cn=engg,ou=testdomain,dc=testdomain,dc=com
#ldapRealm.contextFactory.url = ldap://ldaphost:389
#ldapRealm.contextFactory.authenticationMechanism = SIMPLE
shiro.loginUrl = /api/login
[urls]
# anon means the access is anonymous.
# authcBasic means Basic Auth Security
# To enfore security, comment the line below and uncomment the next one
/api/version = anon
/** = anon
#/** = authcBasic
#/** = authc

View file

@ -62,4 +62,5 @@
# export ZEPPELIN_SPARK_USEHIVECONTEXT # Use HiveContext instead of SQLContext if set true. true by default.
# export ZEPPELIN_SPARK_CONCURRENTSQL # Execute multiple SQL concurrently if set true. false by default.
# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default.
# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000

View file

@ -225,5 +225,11 @@
<description>Anonymous user allowed by default</description>
</property>
<property>
<name>zeppelin.websocket.max.text.message.size</name>
<value>1024000</value>
<description>Size in characters of the maximum text message to be received by websocket. Defaults to 1024000</description>
</property>
</configuration>

View file

@ -231,6 +231,12 @@ You can configure Zeppelin with both **environment variables** in `conf/zeppelin
<td>interpreter</td>
<td>Zeppelin interpreter directory</td>
</tr>
<tr>
<td>ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE</td>
<td>zeppelin.websocket.max.text.message.size</td>
<td>1024000</td>
<td>Size in characters of the maximum text message to be received by websocket.</td>
</tr>
</table>
Maybe you need to configure individual interpreter. If so, please check **Interpreter** section in Zeppelin documentation.

View file

@ -506,7 +506,7 @@
<profile>
<id>spark-1.6</id>
<properties>
<spark.version>1.6.0</spark.version>
<spark.version>1.6.1</spark.version>
<py4j.version>0.9</py4j.version>
<akka.group>com.typesafe.akka</akka.group>
<akka.version>2.3.11</akka.version>
@ -715,6 +715,45 @@
</repositories>
</profile>
<profile>
<id>mapr51</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<hadoop.version>2.7.0-mapr-1602</hadoop.version>
<yarn.version>2.7.0-mapr-1602</yarn.version>
<jets3t.version>0.9.3</jets3t.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5-mapr-1503</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots><enabled>false</enabled></snapshots>
<releases><enabled>true</enabled></releases>
</repository>
</repositories>
</profile>
<profile>
<id>yarn</id>
<dependencies>

View file

@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.rest;
import org.apache.shiro.authc.*;
import org.apache.shiro.session.Session;
import org.apache.shiro.subject.Subject;
import org.apache.zeppelin.server.JsonResponse;
import org.apache.zeppelin.ticket.TicketContainer;
import org.apache.zeppelin.utils.SecurityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.FormParam;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Response;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
/**
* Created for org.apache.zeppelin.rest.message on 17/03/16.
*/
@Path("/login")
@Produces("application/json")
public class LoginRestApi {
private static final Logger LOG = LoggerFactory.getLogger(LoginRestApi.class);
/**
* Required by Swagger.
*/
public LoginRestApi() {
super();
}
/**
* Post Login
* Returns userName & password
* for anonymous access, username is always anonymous.
* After getting this ticket, access through websockets become safe
*
* @return 200 response
*/
@POST
public Response postLogin(@FormParam("userName") String userName,
@FormParam("password") String password) {
JsonResponse response = null;
// ticket set to anonymous for anonymous user. Simplify testing.
Subject currentUser = org.apache.shiro.SecurityUtils.getSubject();
if (!currentUser.isAuthenticated()) {
try {
UsernamePasswordToken token = new UsernamePasswordToken(userName, password);
// token.setRememberMe(true);
currentUser.login(token);
HashSet<String> roles = SecurityUtils.getRoles();
String principal = SecurityUtils.getPrincipal();
String ticket;
if ("anonymous".equals(principal))
ticket = "anonymous";
else
ticket = TicketContainer.instance.getTicket(principal);
Map<String, String> data = new HashMap<>();
data.put("principal", principal);
data.put("roles", roles.toString());
data.put("ticket", ticket);
response = new JsonResponse(Response.Status.OK, "", data);
//if no exception, that's it, we're done!
} catch (UnknownAccountException uae) {
//username wasn't in the system, show them an error message?
LOG.error("Exception in login: ", uae);
} catch (IncorrectCredentialsException ice) {
//password didn't match, try again?
LOG.error("Exception in login: ", ice);
} catch (LockedAccountException lae) {
//account for that username is locked - can't login. Show them a message?
LOG.error("Exception in login: ", lae);
} catch (AuthenticationException ae) {
//unexpected condition - error?
LOG.error("Exception in login: ", ae);
}
}
if (response == null) {
response = new JsonResponse(Response.Status.FORBIDDEN, "", "");
}
LOG.warn(response.toString());
return response.build();
}
}

View file

@ -173,8 +173,9 @@ public class ZeppelinServer extends Application {
private static ServletContextHandler setupNotebookServer(ZeppelinConfiguration conf) {
notebookWsServer = new NotebookServer();
String maxTextMessageSize = conf.getWebsocketMaxTextMessageSize();
final ServletHolder servletHolder = new ServletHolder(notebookWsServer);
servletHolder.setInitParameter("maxTextMessageSize", "1024000");
servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize);
final ServletContextHandler cxfContext = new ServletContextHandler(
ServletContextHandler.SESSIONS);
@ -289,6 +290,9 @@ public class ZeppelinServer extends Application {
SecurityRestApi securityApi = new SecurityRestApi();
singletons.add(securityApi);
LoginRestApi loginRestApi = new LoginRestApi();
singletons.add(loginRestApi);
ConfigurationsRestApi settingsApi = new ConfigurationsRestApi(notebook);
singletons.add(settingsApi);

View file

@ -20,7 +20,11 @@ angular.module('zeppelinWebApp').controller('HomeCtrl', function($scope, noteboo
vm.arrayOrderingSrv = arrayOrderingSrv;
vm.notebookHome = false;
vm.staticHome = false;
if ($rootScope.ticket !== undefined) {
vm.staticHome = false;
} else {
vm.staticHome = true;
}
$scope.isReloading = false;

View file

@ -686,7 +686,11 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl',
}).
error(function(data, status, headers, config) {
console.log('Error %o %o', status, data.message);
alert(data.message);
BootstrapDialog.alert({
closable: true,
title: 'Insufficient privileges',
message: data.message
});
});
};

View file

@ -0,0 +1,42 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict';
angular.module('zeppelinWebApp').controller('LoginCtrl',
function($scope, $rootScope, $http, $httpParamSerializer, baseUrlSrv) {
$scope.loginParams = {};
$scope.login = function() {
$http({
method: 'POST',
url: baseUrlSrv.getRestApiBase() + '/login',
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
},
data: $httpParamSerializer({
'userName': $scope.loginParams.userName,
'password': $scope.loginParams.password
})
}).then(function successCallback(response) {
$rootScope.ticket = response.data.body;
angular.element('#loginModal').modal('toggle');
$rootScope.$broadcast('loginSuccess', true);
}, function errorCallback(errorResponse) {
$scope.loginParams.errorText = 'The username and password that you entered don\'t match.';
});
};
}
);

View file

@ -0,0 +1,51 @@
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div id="loginModal" class="modal fade" role="dialog"
tabindex='-1'>
<div class="modal-dialog">
<!-- Modal content-->
<div class="modal-content" id="NoteImportCtrl" ng-init="NoteImportInit">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal">&times;</button>
<h4 class="modal-title">Login</h4>
</div>
<div class="modal-body">
<div class="form-group" ng-show="loginParams.errorText">
<div class="alert alert-danger">{{loginParams.errorText}}</div>
</div>
<div class="form-group">
<label for="userName">User Name</label>
<input placeholder="User Name" type="text" class="form-control" id="userName"
ng-keypress="loginParams.errorText = ''"
ng-model="loginParams.userName">
</div>
<div class="form-group">
<label for="password">Password</label>
<input placeholder="Password" type="password" class="form-control" id="password"
ng-enter="login()"
ng-keypress="loginParams.errorText = ''"
ng-model="loginParams.password">
</div>
</div>
<div class="modal-footer">
<div>
<button type="button" class="btn btn-default btn-primary" ng-click="login()">Login</button>
</div>
</div>
</div>
</div>
</div>

View file

@ -18,13 +18,22 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
$location, notebookListDataFactory, websocketMsgSrv, arrayOrderingSrv) {
/** Current list of notes (ids) */
$scope.showLoginWindow = function() {
setTimeout(function() {
angular.element('#userName').focus();
}, 500);
};
var vm = this;
vm.notes = notebookListDataFactory;
vm.connected = websocketMsgSrv.isConnected();
vm.websocketMsgSrv = websocketMsgSrv;
vm.arrayOrderingSrv = arrayOrderingSrv;
$rootScope.fullUsername = $rootScope.ticket.principal;
$rootScope.truncatedUsername = $rootScope.ticket.principal;
if ($rootScope.ticket) {
$rootScope.fullUsername = $rootScope.ticket.principal;
$rootScope.truncatedUsername = $rootScope.ticket.principal;
}
var MAX_USERNAME_LENGTH=16;
angular.element('#notebook-list').perfectScrollbar({suppressScrollX: true});
@ -47,14 +56,21 @@ angular.module('zeppelinWebApp').controller('NavCtrl', function($scope, $rootSco
});
$scope.checkUsername = function () {
if($rootScope.ticket.principal.length <= MAX_USERNAME_LENGTH) {
$rootScope.truncatedUsername=$rootScope.ticket.principal;
if ($rootScope.ticket) {
if ($rootScope.ticket.principal.length <= MAX_USERNAME_LENGTH) {
$rootScope.truncatedUsername = $rootScope.ticket.principal;
}
else {
$rootScope.truncatedUsername=$rootScope.ticket.principal.substr(0,MAX_USERNAME_LENGTH)+'..';
else {
$rootScope.truncatedUsername = $rootScope.ticket.principal.substr(0, MAX_USERNAME_LENGTH) + '..';
}
}
};
$scope.$on('loginSuccess', function(event, param) {
$scope.checkUsername();
loadNotes();
});
$scope.search = function() {
$location.url(/search/ + $scope.searchTerm);
};

View file

@ -80,6 +80,9 @@ limitations under the License.
<span ng-show="navbar.connected" ng-if="ticket.principal != 'anonymous' " tooltip-placement="bottom" tooltip="{{fullUsername}}">{{truncatedUsername}}</span>
<span ng-show="!navbar.connected">Disconnected</span>
</li>
<li ng-if="!ticket">
<button class="btn btn-default" data-toggle="modal" data-target="#loginModal" ng-click="showLoginWindow()" style="margin-left: 10px">Login</button>
</li>
</ul>
</div>
</div>

View file

@ -28,9 +28,15 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope,
});
websocketCalls.sendNewEvent = function(data) {
data.principal = $rootScope.ticket.principal;
data.ticket = $rootScope.ticket.ticket;
data.roles = $rootScope.ticket.roles;
if ($rootScope.ticket !== undefined) {
data.principal = $rootScope.ticket.principal;
data.ticket = $rootScope.ticket.ticket;
data.roles = $rootScope.ticket.roles;
} else {
data.principal = '';
data.ticket = '';
data.roles = '';
}
console.log('Send >> %o, %o, %o, %o, %o', data.op, data.principal, data.ticket, data.roles, data);
websocketCalls.ws.send(JSON.stringify(data));
};
@ -54,7 +60,11 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope,
} else if (op === 'NOTES_INFO') {
$rootScope.$broadcast('setNoteMenu', data.notes);
} else if (op === 'AUTH_INFO') {
alert(data.info.toString());
BootstrapDialog.alert({
closable: true,
title: 'Insufficient privileges',
message: data.info.toString()
});
} else if (op === 'PARAGRAPH') {
$rootScope.$broadcast('updateParagraph', data);
} else if (op === 'PARAGRAPH_APPEND_OUTPUT') {

View file

@ -79,6 +79,9 @@ limitations under the License.
<div ng-controller="NoteImportCtrl as noteimportctrl">
<div id="note-import-container" ng-include src="'components/noteName-import/note-import-dialog.html'"></div>
</div>
<div ng-controller="LoginCtrl as noteimportctrl">
<div id="login-container" ng-include src="'components/login/login.html'"></div>
</div>
<!-- build:js(.) scripts/oldieshim.js -->
<!--[if lt IE 9]>
<script src="bower_components/es5-shim/es5-shim.js"></script>
@ -155,6 +158,7 @@ limitations under the License.
<script src="components/browser-detect/browserDetect.service.js"></script>
<script src="components/saveAs/saveAs.service.js"></script>
<script src="components/searchService/search.service.js"></script>
<script src="components/login/login.controller.js"></script>
<!-- endbuild -->
</body>
</html>

View file

@ -387,6 +387,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return Arrays.asList(getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).toLowerCase().split(","));
}
public String getWebsocketMaxTextMessageSize() {
return getString(ConfVars.ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
}
public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
ConfigurationKeyPredicate predicate) {
Map<String, String> configurations = new HashMap<>();
@ -497,7 +501,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
// Allows a way to specify a ',' separated list of allowed origins for rest and websockets
// i.e. http://localhost:8080
ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true);
ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true),
ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000");
private String varName;
@SuppressWarnings("rawtypes")

View file

@ -627,7 +627,8 @@ public class InterpreterFactory {
public void removeNoteInterpreterSettingBinding(String noteId) {
synchronized (interpreterSettings) {
List<String> settingIds = interpreterBindings.remove(noteId);
List<String> settingIds = (interpreterBindings.containsKey(noteId) ?
interpreterBindings.remove(noteId) : Collections.<String>emptyList());
for (String settingId : settingIds) {
this.removeInterpretersForNote(get(settingId), noteId);
}