Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[ZEPPELIN-2261]. Support to connect with livy through https

Browse source
This commit is contained in:
Jeff Zhang 2017-03-15 15:06:21 +08:00
parent 1972a58620
commit 53230c3e55
2 changed files with 69 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
docs/interpreter/livy.md
View file

@ -130,6 +130,16 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory`
<td></td>
<td>Adding extra libraries to livy interpreter</td>
</tr>
<tr>
<td>zeppelin.livy.ssl.trustStore</td>
<td></td>
<td>client trustStore file. Used when livy ssl is enabled</td>
</tr>
<tr>
<td>zeppelin.livy.ssl.trustStorePassword</td>
<td></td>
<td>password for trustStore file. Used when livy ssl is enabled</td>
</tr>
</table>
**We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.**

73
livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
View file

@ -21,6 +21,10 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.impl.client.HttpClients;
import org.apache.zeppelin.interpreter.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -28,11 +32,16 @@ import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.security.kerberos.client.KerberosRestTemplate;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import javax.net.ssl.SSLContext;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.KeyStore;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -146,7 +155,7 @@ public abstract class BaseLivyInterprereter extends Interpreter {
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
if (livyVersion.isCancelSupported()) {
@ -312,12 +321,12 @@ public abstract class BaseLivyInterprereter extends Interpreter {
} else {
//TODO(zjffdu) support other types of data (like json, image and etc)
String result = stmtInfo.output.data.plain_text;
// check table magic result first
if (stmtInfo.output.data.application_livy_table_json != null) {
StringBuilder outputBuilder = new StringBuilder();
boolean notFirstColumn = false;
for (Map header : stmtInfo.output.data.application_livy_table_json.headers) {
if (notFirstColumn) {
outputBuilder.append("\t");
@ -325,17 +334,17 @@ public abstract class BaseLivyInterprereter extends Interpreter {
outputBuilder.append(header.get("name"));
notFirstColumn = true;
}
outputBuilder.append("\n");
for (List<Object> row : stmtInfo.output.data.application_livy_table_json.records) {
outputBuilder.append(StringUtils.join(row, "\t"));
outputBuilder.append("\n");
}
outputBuilder.append("\n");
}
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.TABLE, outputBuilder.toString());
} else if (stmtInfo.output.data.image_png != null) {
InterpreterResult.Type.TABLE, outputBuilder.toString());
} else if (stmtInfo.output.data.image_png != null) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
InterpreterResult.Type.IMG, (String) stmtInfo.output.data.image_png);
InterpreterResult.Type.IMG, (String) stmtInfo.output.data.image_png);
} else if (result != null) {
result = result.trim();
if (result.startsWith("<link")
@ -378,13 +387,49 @@ public abstract class BaseLivyInterprereter extends Interpreter {
callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId + "/cancel", "POST");
}
private RestTemplate getRestTemplate() {
private RestTemplate getRestTemplate() throws LivyException {
HttpClient httpClient = null;
if (livyURL.startsWith("https:")) {
String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore");
String password = property.getProperty("zeppelin.livy.ssl.trustStorePassword");
FileInputStream inputStream = null;
try {
inputStream = new FileInputStream(keystoreFile);
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(new FileInputStream(keystoreFile), password.toCharArray());
SSLContext sslContext = SSLContexts.custom()
.loadTrustMaterial(trustStore)
.build();
SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext);
httpClient = HttpClients.custom().setSSLSocketFactory(csf).build();
} catch (Exception e) {
throw new LivyException("Failed to create SSL HttpClient", e);
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
LOGGER.error("Failed to close keystore file", e);
}
}
}
}
String keytabLocation = property.getProperty("zeppelin.livy.keytab");
String principal = property.getProperty("zeppelin.livy.principal");
if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) {
return new KerberosRestTemplate(keytabLocation, principal);
if (httpClient == null) {
return new KerberosRestTemplate(keytabLocation, principal);
} else {
return new KerberosRestTemplate(keytabLocation, principal, httpClient);
}
}
if (httpClient == null) {
return new RestTemplate();
} else {
return new RestTemplate(new HttpComponentsClientHttpRequestFactory(httpClient));
}
return new RestTemplate();
}
private String callRestAPI(String targetURL, String method) throws LivyException {
@ -581,11 +626,11 @@ public abstract class BaseLivyInterprereter extends Interpreter {
@SerializedName("application/vnd.livy.table.v1+json")
public TableMagic application_livy_table_json;
}
private static class TableMagic {
@SerializedName("headers")
List<Map> headers;
@SerializedName("data")
List<List> records;
}