Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
tinkoff-dwh 2017-10-16 11:22:51 +05:00
commit 1932202f47
11 changed files with 70 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
conf/zeppelin-site.xml.template
View file

@ -138,6 +138,16 @@
</property>
-->
<!-- Optional override to control which signature algorithm should be used to sign AWS requests -->
<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. -->
<!--
<property>
<name>zeppelin.notebook.s3.signerOverride</name>
<value>S3SignerType</value>
<description>optional override to control which signature algorithm should be used to sign AWS requests</description>
</property>
-->
<!-- If using Azure for storage use the following settings -->
<!--
<property>

6
docs/setup/operation/configuration.md
View file

@ -215,6 +215,12 @@ If both are defined, then the **environment variables** will take priority.
<td>false</td>
<td>Save notebooks to S3 with server-side encryption enabled</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE</h6></td>
<td><h6 class="properties">zeppelin.notebook.s3.signerOverride</h6></td>
<td></td>
<td>Optional override to control which signature algorithm should be used to sign AWS requests</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</h6></td>
<td><h6 class="properties">zeppelin.notebook.azure.connectionString</h6></td>

8
interpreter-parent/pom.xml
View file

@ -78,6 +78,14 @@
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>none</phase>
<configuration>
<skip>true</skip>
</configuration>
</execution>
<execution>
<id>copy-interpreter-dependencies</id>
<phase>package</phase>

2
python/src/main/java/org/apache/zeppelin/python/IPythonClient.java
View file

@ -96,7 +96,7 @@ public class IPythonClient {
LOGGER.debug("Interpreter Streaming Output: " + executeResponse.getOutput());
if (isPreviousOutputImage) {
// add '\n' when switch from image to text
interpreterOutput.write("\n".getBytes());
interpreterOutput.write("\n%text ".getBytes());
}
isPreviousOutputImage = false;
interpreterOutput.write(executeResponse.getOutput().getBytes());

1
python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
View file

@ -286,6 +286,7 @@ public class IPythonInterpreter extends Interpreter implements ExecuteResultHand
} else {
envs.put("PYTHONPATH", additionalPythonPath);
}
LOGGER.info("PYTHONPATH:" + envs.get("PYTHONPATH"));
return envs;
}

12
python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java
View file

@ -284,12 +284,24 @@ public class IPythonInterpreterTest {
// the order of IMAGE and TEXT is not determined
// check there must be one IMAGE output
boolean hasImageOutput = false;
boolean hasLineText = false;
boolean hasFigureText = false;
for (InterpreterResultMessage msg : interpreterResultMessages) {
if (msg.getType() == InterpreterResult.Type.IMG) {
hasImageOutput = true;
}
if (msg.getType() == InterpreterResult.Type.TEXT
&& msg.getData().contains("matplotlib.lines.Line2D")) {
hasLineText = true;
}
if (msg.getType() == InterpreterResult.Type.TEXT
&& msg.getData().contains("matplotlib.figure.Figure")) {
hasFigureText = true;
}
}
assertTrue("No Image Output", hasImageOutput);
assertTrue("No Line Text", hasLineText);
assertTrue("No Figure Text", hasFigureText);
// bokeh
// bokeh initialization

4
spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java
View file

@ -146,7 +146,7 @@ public class IPySparkInterpreterTest {
List<InterpreterResultMessage> interpreterResultMessages = null;
try {
interpreterResultMessages = context2.out.getInterpreterResultMessages();
assertTrue(interpreterResultMessages.get(0).getData().contains("cancelled"));
assertTrue(interpreterResultMessages.get(0).getData().contains("KeyboardInterrupt"));
} catch (IOException e) {
e.printStackTrace();
}
@ -180,7 +180,7 @@ public class IPySparkInterpreterTest {
"ssc.start()\n" +
"time.sleep(6)\n" +
"ssc.stop(stopSparkContext=False, stopGraceFully=True)", context);
Thread.sleep(100);
Thread.sleep(1000);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
interpreterResultMessages = context.out.getInterpreterResultMessages();
assertEquals(1, interpreterResultMessages.size());

5
zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -383,6 +383,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE);
}
public String getS3SignerOverride() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE);
}
public String getMongoUri() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_URI);
}
@ -654,6 +658,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE("zeppelin.notebook.s3.signerOverride", null),
ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),

2
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -357,7 +357,7 @@ public class NotebookServer extends WebSocketServlet
break;
}
} catch (Exception e) {
LOG.error("Can't handle message", e);
LOG.error("Can't handle message: " + msg, e);
}
}

1
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java
View file

@ -246,7 +246,6 @@ public class RemoteInterpreterEventPoller extends Thread {
listener.onParaInfosReceived(noteId, paraId, settingId, paraInfos);
}
}
logger.debug("Event from remote process {}", event.getType());
} catch (Exception e) {
logger.error("Can't handle event " + event, e);
}

33
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -42,6 +42,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.AmazonClientException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.ClientConfigurationFactory;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
@ -94,33 +96,30 @@ public class S3NotebookRepo implements NotebookRepo {
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
CryptoConfiguration cryptoConf = null;
CryptoConfiguration cryptoConf = new CryptoConfiguration();
String keyRegion = conf.getS3KMSKeyRegion();
if (StringUtils.isNotBlank(keyRegion)) {
cryptoConf = new CryptoConfiguration();
cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
}
ClientConfiguration cliConf = createClientConfiguration();
// see if we should be encrypting data in S3
String kmsKeyID = conf.getS3KMSKeyID();
if (kmsKeyID != null) {
// use the AWS KMS to encrypt data
KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
if (cryptoConf != null) {
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cryptoConf);
} else {
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
}
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
}
else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
// use a custom encryption materials provider class
EncryptionMaterialsProvider emp = createCustomProvider(conf);
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
}
else {
// regular S3
this.s3client = new AmazonS3Client(credentialsProvider);
this.s3client = new AmazonS3Client(credentialsProvider, cliConf);
}
// set S3 endpoint to use
@ -154,6 +153,22 @@ public class S3NotebookRepo implements NotebookRepo {
return emp;
}
/**
* Create AWS client configuration and return it.
* @return AWS client configuration
*/
private ClientConfiguration createClientConfiguration() {
ClientConfigurationFactory configFactory = new ClientConfigurationFactory();
ClientConfiguration config = configFactory.getConfig();
String s3SignerOverride = conf.getS3SignerOverride();
if (StringUtils.isNotBlank(s3SignerOverride)) {
config.setSignerOverride(s3SignerOverride);
}
return config;
}
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<>();