Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Support Ceph as a notebook storage

Browse source
This commit is contained in:
Keiji Yoshida 2017-09-25 21:03:54 +09:00
parent 84e9bd96d2
commit c66cb638fd
4 changed files with 45 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
conf/zeppelin-site.xml.template
View file

@ -138,6 +138,16 @@
</property>
-->
<!-- Optional override to control which signature algorithm should be used to sign AWS requests -->
<!-- Set this property to "S3SignerType" if your AWS S3 compatible APIs support only AWS Signature Version 2 such as Ceph. -->
<!--
<property>
<name>zeppelin.notebook.s3.signerOverride</name>
<value>S3SignerType</value>
<description>optional override to control which signature algorithm should be used to sign AWS requests</description>
</property>
-->
<!-- If using Azure for storage use the following settings -->
<!--
<property>

6
docs/setup/operation/configuration.md
View file

@ -203,6 +203,12 @@ If both are defined, then the **environment variables** will take priority.
<td>false</td>
<td>Save notebooks to S3 with server-side encryption enabled</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE</h6></td>
<td><h6 class="properties">zeppelin.notebook.s3.signerOverride</h6></td>
<td></td>
<td>Optional override to control which signature algorithm should be used to sign AWS requests</td>
</tr>
<tr>
<td><h6 class="properties">ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING</h6></td>
<td><h6 class="properties">zeppelin.notebook.azure.connectionString</h6></td>

5
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -383,6 +383,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE);
}
public String getS3SignerOverride() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE);
}
public String getMongoUri() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_URI);
}
@ -649,6 +653,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE("zeppelin.notebook.s3.signerOverride", null),
ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),

33
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
View file

@ -42,6 +42,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.amazonaws.AmazonClientException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.ClientConfigurationFactory;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3;
@ -94,33 +96,30 @@ public class S3NotebookRepo implements NotebookRepo {
// always use the default provider chain
AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
CryptoConfiguration cryptoConf = null;
CryptoConfiguration cryptoConf = new CryptoConfiguration();
String keyRegion = conf.getS3KMSKeyRegion();
if (StringUtils.isNotBlank(keyRegion)) {
cryptoConf = new CryptoConfiguration();
cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
}
ClientConfiguration cliConf = createClientConfiguration();
// see if we should be encrypting data in S3
String kmsKeyID = conf.getS3KMSKeyID();
if (kmsKeyID != null) {
// use the AWS KMS to encrypt data
KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
if (cryptoConf != null) {
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cryptoConf);
} else {
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
}
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
}
else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
// use a custom encryption materials provider class
EncryptionMaterialsProvider emp = createCustomProvider(conf);
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp);
this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
}
else {
// regular S3
this.s3client = new AmazonS3Client(credentialsProvider);
this.s3client = new AmazonS3Client(credentialsProvider, cliConf);
}
// set S3 endpoint to use
@ -154,6 +153,22 @@ public class S3NotebookRepo implements NotebookRepo {
return emp;
}
/**
* Create AWS client configuration and return it.
* @return AWS client configuration
*/
private ClientConfiguration createClientConfiguration() {
ClientConfigurationFactory configFactory = new ClientConfigurationFactory();
ClientConfiguration config = configFactory.getConfig();
String s3SignerOverride = conf.getS3SignerOverride();
if (StringUtils.isNotBlank(s3SignerOverride)) {
config.setSignerOverride(s3SignerOverride);
}
return config;
}
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
List<NoteInfo> infos = new LinkedList<>();