[Zeppelin-1611] - Support PAM (System User) Authentication

This PR adds [PAM](https://en.wikipedia.org/wiki/Pluggable_authentication_module) authentication support based on the introduced Shiro security implementation. With PAM support system users have immediate access to a secured Zeppelin instance.

Feature

* [x] - Create PAM realm
* [x] - Create test for PAM authentication
* [x] - Test with running Zeppelin instance

[ZEPPELIN-1611](https://issues.apache.org/jira/browse/ZEPPELIN-1611])

`PamRealmTest` executes an automated test if the environment variables `PAM_USER` and `PAM_PASS` are set. This should be set to system username and password.
The test also includes a main function to manually execute the test. Setting the environment variables for example on MacOS for your IDE use `launchctl setenv PAM_USER user` and `launchctl setenv PAM_PASS xxxxx`, the test can then be run from your IDE.

* Does the licenses files need update? Yes
* Is there breaking changes for older versions? No
* Does this needs documentation? Yes

conf/shiro.ini.template

@@ -45,6 +45,10 @@ user3 = password4, role2
 #ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
 #ldapRealm.contextFactory.authenticationMechanism = SIMPLE
 
+### A sample PAM configuration
+#pamRealm=org.apache.zeppelin.realm.PamRealm
+#pamRealm.service=sshd
+
 ### A sample for configuring ZeppelinHub Realm
 #zeppelinHubRealm = org.apache.zeppelin.realm.ZeppelinHubRealm
 ## Url of ZeppelinHub

docs/security/shiroauthentication.md

@@ -147,6 +147,19 @@ ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM
 ldapRealm.contextFactory.authenticationMechanism = SIMPLE
 ```
 
+### PAM
+[PAM](https://en.wikipedia.org/wiki/Pluggable_authentication_module) authentication support allows the reuse of existing authentication 
+moduls on the host where Zeppelin is running. On a typical system modules are configured per service for example sshd, passwd, etc. under `/etc/pam.d/`. You can
+either reuse one of these services or create your own for Zeppelin. Activiting PAM authentication requires two parameters:
+ 1. realm: The Shiro realm being used
+ 2. service: The service configured under `/etc/pam.d/` to be used. The name here needs to be the same as the file name under `/etc/pam.d/`
+ 
+```
+[main]
+ pamRealm=org.apache.zeppelin.realm.PamRealm
+ pamRealm.service=sshd
+```
+
 ### ZeppelinHub
 [ZeppelinHub](https://www.zeppelinhub.com) is a service that synchronize your Apache Zeppelin notebooks and enables you to collaborate easily.
 

zeppelin-distribution/src/bin_license/LICENSE

@@ -163,6 +163,7 @@ The following components are provided under Apache License.
     (Apache 2.0) tez-runtime-internals (org.apache.tez:tez-runtime-internals:0.7.0 - http://tez.apache.org)
     (Apache 2.0) tez-mapreduce (org.apache.tez:tez-mapreduce:0.7.0 - http://tez.apache.org)
     (Apache 2.0) tez-yarn-timeline-history-with-acls (org.apache.tez:tez-yarn-timeline-history-with-acls:0.7.0 - http://tez.apache.org)
+    (Apache 2.0) jna (net.java.dev.jna:jna:4.1.0 https://github.com/java-native-access/jna)
 
 ========================================================================
 MIT licenses
@@ -208,7 +209,8 @@ The following components are provided under the MIT License.
     (The MIT License) JUL to SLF4J bridge (org.slf4j:jul-to-slf4j:1.7.16 - http://www.slf4j.org)
     (The MIT License) angular-resource (angular-resource - https://github.com/angular/angular.js/tree/master/src/ngResource)
     (The MIT License) minimal-json (com.eclipsesource.minimal-json:minimal-json:0.9.4 - https://github.com/ralfstx/minimal-json)
-    (The MIT License) pyrolite (net.razorvine:pyrolite:4.9) - https://github.com/irmen/Pyrolite/blob/v4.9/LICENSE
+    (The MIT License) pyrolite (net.razorvine:pyrolite:4.9) - https://github.com/irmen/Pyrolite/blob/v4.9/LICENSE)
+    (The MIT License) libpam4j (org.kohsuke:libpam4j:1.8 https://github.com/kohsuke/libpam4j/blob/master/src/main/java/org/jvnet/libpam/PAM.java)
 
 ========================================================================
 BSD-style licenses

zeppelin-server/pom.xml

@@ -319,6 +319,10 @@
           <groupId>org.eclipse.jetty.websocket</groupId>
           <artifactId>websocket-client</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>net.java.dev.jna</groupId>
+          <artifactId>jna</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -353,6 +357,22 @@
       <groupId>org.apache.shiro</groupId>
       <artifactId>shiro-web</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.kohsuke</groupId>
+      <artifactId>libpam4j</artifactId>
+      <version>1.8</version>
+      <exclusions>
+        <exclusion>
+          <groupId>net.java.dev.jna</groupId>
+          <artifactId>jna</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>net.java.dev.jna</groupId>
+      <artifactId>jna</artifactId>
+      <version>4.1.0</version>
+    </dependency>
   </dependencies>
 
   <build>

zeppelin-server/src/main/java/org/apache/zeppelin/realm/PamRealm.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.realm;
+
+import org.apache.shiro.authc.AuthenticationException;
+import org.apache.shiro.authc.AuthenticationInfo;
+import org.apache.shiro.authc.AuthenticationToken;
+import org.apache.shiro.authc.SimpleAuthenticationInfo;
+import org.apache.shiro.authc.UsernamePasswordToken;
+import org.apache.shiro.authz.AuthorizationInfo;
+import org.apache.shiro.authz.SimpleAuthorizationInfo;
+import org.apache.shiro.crypto.hash.DefaultHashService;
+import org.apache.shiro.crypto.hash.Hash;
+import org.apache.shiro.crypto.hash.HashRequest;
+import org.apache.shiro.crypto.hash.HashService;
+import org.apache.shiro.realm.AuthorizingRealm;
+import org.apache.shiro.subject.PrincipalCollection;
+import org.jvnet.libpam.PAM;
+import org.jvnet.libpam.PAMException;
+import org.jvnet.libpam.UnixUser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * An {@code AuthorizingRealm} base on libpam4j.
+ */
+public class PamRealm extends AuthorizingRealm {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ZeppelinHubRealm.class);
+
+  private String service;
+
+  @Override
+  protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principals) {
+    Set<String> roles = new LinkedHashSet<>();
+
+    UserPrincipal user = principals.oneByType(UserPrincipal.class);
+
+    if (user != null){
+      roles.addAll(user.getUnixUser().getGroups());
+    }
+
+    return new SimpleAuthorizationInfo(roles);
+  }
+
+  @Override
+  protected AuthenticationInfo doGetAuthenticationInfo(AuthenticationToken token)
+      throws AuthenticationException {
+
+    UsernamePasswordToken userToken = (UsernamePasswordToken) token;
+    UnixUser user;
+
+    try {
+      user = (new PAM(this.getService()))
+          .authenticate(userToken.getUsername(), new String(userToken.getPassword()));
+    } catch (PAMException e) {
+      throw new AuthenticationException("Authentication failed for PAM.", e);
+    }
+
+    return new SimpleAuthenticationInfo(
+        new UserPrincipal(user),
+        userToken.getCredentials(),
+        getName());
+  }
+
+  public String getService() {
+    return service;
+  }
+
+  public void setService(String service) {
+    this.service = service;
+  }
+
+}

zeppelin-server/src/main/java/org/apache/zeppelin/realm/UserPrincipal.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.realm;
+
+import org.jvnet.libpam.UnixUser;
+
+import java.security.Principal;
+
+/**
+ * A {@code java.security.Principal} implememtation for use with Shiro {@code PamRealm}
+ */
+public class UserPrincipal implements Principal {
+  private final UnixUser userName;
+
+  public UserPrincipal(UnixUser userName) {
+    this.userName = userName;
+  }
+
+  @Override
+  public String getName() {
+    return userName.getUserName();
+  }
+
+  public UnixUser getUnixUser() {
+    return userName;
+  }
+
+  @Override
+  public String toString() {
+    return String.valueOf(userName.getUserName());
+  }
+}

zeppelin-server/src/test/java/org/apache/zeppelin/realm/PamRealmTest.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.realm;
+
+import org.apache.shiro.authc.AuthenticationInfo;
+import org.apache.shiro.authc.UsernamePasswordToken;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * The test will only be executed if the environment variables PAM_USER and PAM_PASS are present. They should
+ * contain username and password of an valid system user to make the test pass. The service needs to be configured
+ * under /etc/pam.d/sshd to resolve and authenticate the system user.
+ *
+ * Contains main() function so the test can be executed manually.
+ *
+ * Set in MacOS to run in IDE(A):
+ * $ launchctl setenv PAM_USER user
+ * $ launchctl setenv PAM_PASS xxxxx
+ */
+public class PamRealmTest {
+
+  @Test
+  public void testDoGetAuthenticationInfo() {
+    PamRealm realm = new PamRealm();
+    realm.setService("sshd");
+
+    String pam_user = System.getenv("PAM_USER");
+    String pam_pass = System.getenv("PAM_PASS");
+    assumeTrue(pam_user != null);
+    assumeTrue(pam_pass != null);
+
+    // mock shiro auth token
+    UsernamePasswordToken authToken = mock(UsernamePasswordToken.class);
+    when(authToken.getUsername()).thenReturn(pam_user);
+    when(authToken.getPassword()).thenReturn(pam_pass.toCharArray());
+    when(authToken.getCredentials()).thenReturn(pam_pass);
+
+    AuthenticationInfo authInfo = realm.doGetAuthenticationInfo(authToken);
+
+    assertTrue(authInfo.getCredentials() != null);
+  }
+
+  public static void main(String[] args) {
+    PamRealmTest test = new PamRealmTest();
+    test.testDoGetAuthenticationInfo();
+  }
+}