mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Merge remote-tracking branch 'upstream/master' into ZEPPELIN-2403
# Conflicts: # spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
This commit is contained in:
Tinkoff DWH
commit
637cb0a155
54 changed files with 1245 additions and 546 deletions
26
.travis.yml
26
.travis.yml
|
|
@ -47,11 +47,11 @@ matrix:
|
|||
|
||||
# Test core modules
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtest='!ZeppelinSparkClusterTest,!org.apache.zeppelin.spark.*' -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtest='!ZeppelinSparkClusterTest,!org.apache.zeppelin.spark.*' -DfailIfNoTests=false"
|
||||
|
||||
# Test selenium with spark module for 1.6.3
|
||||
- jdk: "oraclejdk7"
|
||||
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
|
||||
|
||||
# Test interpreter modules
|
||||
- jdk: "oraclejdk7"
|
||||
|
|
@ -59,27 +59,29 @@ matrix:
|
|||
|
||||
# Test spark module for 2.1.0 with scala 2.11, livy
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 2.0.2 with scala 2.11
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.6.3 with scala 2.10
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test spark module for 1.6.3 with scala 2.11
|
||||
- jdk: "oraclejdk7"
|
||||
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
|
||||
|
||||
# Test python/pyspark with python 2
|
||||
- jdk: "oraclejdk7"
|
||||
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
# Test python/pyspark with python 2, livy 0.2
|
||||
- sudo: required
|
||||
jdk: "oraclejdk7"
|
||||
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Plivy-0.2" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
|
||||
# Test python/pyspark with python 3
|
||||
- jdk: "oraclejdk7"
|
||||
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
# Test python/pyspark with python 3, livy 0.3
|
||||
- sudo: required
|
||||
jdk: "oraclejdk7"
|
||||
env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false"
|
||||
|
||||
before_install:
|
||||
# check files included in commit range, clear bower_components if a bower.json file has changed.
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 8.1 KiB |
|
|
@ -27,44 +27,35 @@ limitations under the License.
|
|||
This document contains instructions about making docker containers for Zeppelin. It mainly provides guidance into how to create, publish and run docker images for zeppelin releases.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Installing Docker
|
||||
You need to [install docker](https://docs.docker.com/engine/installation/) on your machine.
|
||||
|
||||
### Creating and Publishing Zeppelin docker image
|
||||
* In order to be able to create and/or publish an image, you need to set the **DockerHub** credentials `DOCKER_USERNAME, DOCKER_PASSWORD, DOCKER_EMAIL` variables as environment variables.
|
||||
|
||||
* To create an image for some release use :
|
||||
`create_release.sh <release-version> <git-tag>`.
|
||||
* To publish the created image use :
|
||||
`publish_release.sh <release-version> <git-tag>`
|
||||
|
||||
### Running a Zeppelin docker image
|
||||
|
||||
* To start Zeppelin, you need to pull the zeppelin release image:
|
||||
```
|
||||
docker pull ${DOCKER_USERNAME}/zeppelin-release:<release-version>
|
||||
|
||||
docker run --rm -it -p 7077:7077 -p 8080:8080 ${DOCKER_USERNAME}/zeppelin-release:<release-version> -c bash
|
||||
```
|
||||
* Then a docker container will start with a Zeppelin release on path :
|
||||
`/usr/local/zeppelin/`
|
||||
|
||||
* Run zeppelin inside docker:
|
||||
```
|
||||
/usr/local/zeppelin/bin/zeppelin.sh
|
||||
```
|
||||
|
||||
* To Run Zeppelin in daemon mode
|
||||
Mounting logs and notebooks zeppelin to folders on your host machine
|
||||
### Running docker image
|
||||
|
||||
```
|
||||
docker run -p 7077:7077 -p 8080:8080 --privileged=true -v $PWD/logs:/logs -v $PWD/notebook:/notebook \
|
||||
-e ZEPPELIN_NOTEBOOK_DIR='/notebook' \
|
||||
-e ZEPPELIN_LOG_DIR='/logs' \
|
||||
-d ${DOCKER_USERNAME}/zeppelin-release:<release-version> \
|
||||
/usr/local/zeppelin/bin/zeppelin.sh
|
||||
docker run -p 8080:8080 --name zeppelin zeppelin:<release-version>
|
||||
```
|
||||
|
||||
|
||||
* Zeppelin will run at `http://localhost:8080`.
|
||||
|
||||
If you want to specify `logs` and `notebook` dir,
|
||||
|
||||
```
|
||||
docker run -p 8080:8080 \
|
||||
-v $PWD/logs:/logs \
|
||||
-v $PWD/notebook:/notebook \
|
||||
-e ZEPPELIN_LOG_DIR='/logs' \
|
||||
-e ZEPPELIN_NOTEBOOK_DIR='/notebook' \
|
||||
--name zeppelin zeppelin:<release-version> # e.g '0.7.1'
|
||||
```
|
||||
|
||||
### Building dockerfile locally
|
||||
|
||||
```
|
||||
cd $ZEPPELIN_HOME
|
||||
cd scripts/docker/zeppelin
|
||||
|
||||
./create-dockerfile.sh <release-version> # e.g '0.7.1'
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,8 @@ def r = HTTP.get(
|
|||
headers: [
|
||||
'Accept':'application/json',
|
||||
//'Authorization:' : g.getProperty('search_auth'),
|
||||
]
|
||||
],
|
||||
ssl : g.getProperty('search_ssl') // assume groovy interpreter property search_ssl = HTTP.getNaiveSSLContext()
|
||||
)
|
||||
//check response code
|
||||
if( r.response.code==200 ) {
|
||||
|
|
@ -76,41 +77,62 @@ g.table(
|
|||
|
||||
* `g.angular(String name)`
|
||||
|
||||
Returns angular object by name. Look up notebook scope first and then global scope.
|
||||
Returns angular object by name. Look up notebook scope first and then global scope.
|
||||
|
||||
|
||||
* `g.angularBind(String name, Object value)`
|
||||
|
||||
Assign a new `value` into angular object `name`
|
||||
|
||||
Assign a new `value` into angular object `name`
|
||||
|
||||
|
||||
* `java.util.Properties g.getProperties()`
|
||||
|
||||
returns all properties defined for this interpreter
|
||||
returns all properties defined for this interpreter
|
||||
|
||||
|
||||
* `String g.getProperty('PROPERTY_NAME')`
|
||||
```groovy
|
||||
g.PROPERTY_NAME
|
||||
g.'PROPERTY_NAME'
|
||||
g['PROPERTY_NAME']
|
||||
g.getProperties().getProperty('PROPERTY_NAME')
|
||||
```
|
||||
```groovy
|
||||
g.PROPERTY_NAME
|
||||
g.'PROPERTY_NAME'
|
||||
g['PROPERTY_NAME']
|
||||
g.getProperties().getProperty('PROPERTY_NAME')
|
||||
```
|
||||
|
||||
All above the accessor to named property defined in groovy interpreter.
|
||||
In this case with name `PROPERTY_NAME`
|
||||
All above the accessor to named property defined in groovy interpreter.
|
||||
In this case with name `PROPERTY_NAME`
|
||||
|
||||
|
||||
* `groovy.xml.MarkupBuilder g.html()`
|
||||
|
||||
Starts or continues rendering of `%angular` to output and returns [groovy.xml.MarkupBuilder](http://groovy-lang.org/processing-xml.html#_markupbuilder)
|
||||
MarkupBuilder is usefull to generate html (xml)
|
||||
Starts or continues rendering of `%angular` to output and returns [groovy.xml.MarkupBuilder](http://groovy-lang.org/processing-xml.html#_markupbuilder)
|
||||
MarkupBuilder is usefull to generate html (xml)
|
||||
|
||||
* `void g.table(obj)`
|
||||
|
||||
starts or continues rendering table rows.
|
||||
starts or continues rendering table rows.
|
||||
|
||||
obj: List(rows) of List(columns) where first line is a header
|
||||
obj: List(rows) of List(columns) where first line is a header
|
||||
|
||||
|
||||
* `g.input(name, value )`
|
||||
|
||||
Creates `text` input with value specified. The parameter `value` is optional.
|
||||
|
||||
* `g.select(name, default, Map<Object, String> options)`
|
||||
|
||||
Creates `select` input with defined options. The parameter `default` is optional.
|
||||
|
||||
```g.select('sex', 'm', ['m':'man', 'w':'woman'])```
|
||||
|
||||
* `g.checkbox(name, Collection checked, Map<Object, String> options)`
|
||||
|
||||
Creates `checkbox` input.
|
||||
|
||||
* `g.get(name, default)`
|
||||
|
||||
Returns interpreter-based variable. Visibility depends on interpreter scope. The parameter `default` is optional.
|
||||
|
||||
* `g.put(name, value)`
|
||||
|
||||
Stores new value into interpreter-based variable. Visibility depends on interpreter scope.
|
||||
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ The JDBC interpreter properties are defined by default like below.
|
|||
<tr>
|
||||
<td>default.precode</td>
|
||||
<td></td>
|
||||
<td>Some SQL which executes while opening connection</td>
|
||||
<td>Some SQL which executes every time after initialization of the interpreter (see [Binding mode](../manual/interpreters.md#interpreter-binding-mode))</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>default.completer.schemaFilters</td>
|
||||
|
|
|
|||
|
|
@ -113,3 +113,11 @@ interpreter.start()
|
|||
The above code will start interpreter thread inside your process. Once the interpreter is started you can configure zeppelin to connect to RemoteInterpreter by checking **Connect to existing process** checkbox and then provide **Host** and **Port** on which interpreter process is listening as shown in the image below:
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/screenshots/existing_interpreter.png" width="450px">
|
||||
|
||||
## Precode
|
||||
|
||||
Snippet of code (language of interpreter) that executes after initialization of the interpreter depends on [Binding mode](#interpreter-binding-mode). To configure add parameter with class of interpreter (`zeppelin.<ClassName>.precode`) except JDBCInterpreter ([JDBC precode](../interpreter/jdbc.md#usage-precode)).
|
||||
|
||||
<img src="../assets/themes/zeppelin/img/screenshots/interpreter_precode.png" width="800px">
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
|
|||
activeDirectoryRealm.url = ldap://ldap.test.com:389
|
||||
activeDirectoryRealm.groupRolesMap = "CN=aGroupName,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"group1"
|
||||
activeDirectoryRealm.authorizationCachingEnabled = false
|
||||
activeDirectoryRealm.principalSuffix = @corp.company.net
|
||||
|
||||
ldapRealm = org.apache.zeppelin.server.LdapGroupRealm
|
||||
# search base for ldap groups (only relevant for LdapGroupRealm):
|
||||
|
|
@ -114,6 +115,7 @@ activeDirectoryRealm.searchBase = CN=Users,DC=SOME_GROUP,DC=COMPANY,DC=COM
|
|||
activeDirectoryRealm.url = ldap://ldap.test.com:389
|
||||
activeDirectoryRealm.groupRolesMap = "CN=aGroupName,OU=groups,DC=SOME_GROUP,DC=COMPANY,DC=COM":"group1"
|
||||
activeDirectoryRealm.authorizationCachingEnabled = false
|
||||
activeDirectoryRealm.principalSuffix = @corp.company.net
|
||||
```
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,16 @@
|
|||
|
||||
import groovy.json.JsonOutput
|
||||
|
||||
import java.security.KeyStore;
|
||||
import javax.net.ssl.KeyManagerFactory;
|
||||
import javax.net.ssl.KeyManager;
|
||||
import javax.net.ssl.TrustManager;
|
||||
import javax.net.ssl.X509TrustManager;
|
||||
import java.security.cert.X509Certificate;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import java.security.SecureRandom;
|
||||
import javax.net.ssl.HttpsURLConnection;
|
||||
|
||||
/**
|
||||
* simple http rest client for groovy
|
||||
* by dlukyanov@ukr.net
|
||||
|
|
@ -44,6 +54,11 @@ public class HTTP{
|
|||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> head(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','HEAD');
|
||||
return send(ctx);
|
||||
}
|
||||
|
||||
public static Map<String,Object> post(Map<String,Object> ctx)throws IOException{
|
||||
ctx.put('method','POST');
|
||||
return send(ctx);
|
||||
|
|
@ -59,6 +74,16 @@ public class HTTP{
|
|||
return send(ctx);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param url string where to send request
|
||||
* @param query Map parameters to append to url
|
||||
* @param method http method to be used in request. standard methods: GET, POST, PUT, DELETE, HEAD
|
||||
* @param headers key-value map with headers that should be sent with request
|
||||
* @param body request body/data to send to url (InputStream, CharSequence, or Map for json and x-www-form-urlencoded context types)
|
||||
* @param encoding encoding name to use to send/receive data - default UTF-8
|
||||
* @param receiver Closure that will be called to receive data from server. Defaults: `HTTP.JSON_RECEIVER` for json content-type and `HTTP.TEXT_RECEIVER` otherwise. Available: `HTTP.FILE_RECEIVER(File)` - stores response to file.
|
||||
* @param ssl javax.net.ssl.SSLContext or String that evaluates the javax.net.ssl.SSLContext. example: send( url:..., ssl: "HTTP.getKeystoreSSLContext('./keystore.jks', 'testpass')" )
|
||||
*/
|
||||
public static Map<String,Object> send(Map<String,Object> ctx)throws IOException{
|
||||
String url = ctx.url;
|
||||
Map<String,String> headers = (Map<String,String>)ctx.headers;
|
||||
|
|
@ -67,6 +92,7 @@ public class HTTP{
|
|||
String encoding = ctx.encoding?:"UTF-8";
|
||||
Closure receiver = (Closure)ctx.receiver;
|
||||
Map<String,String> query = (Map<String,String>)ctx.query;
|
||||
Object sslCtxObj= ctx.ssl;
|
||||
|
||||
//copy context and set default values
|
||||
ctx = [:] + ctx;
|
||||
|
|
@ -78,14 +104,28 @@ public class HTTP{
|
|||
}
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
|
||||
if(sslCtxObj!=null && connection instanceof HttpsURLConnection){
|
||||
SSLContext sslCtx = null;
|
||||
if(sslCtxObj instanceof SSLContext){
|
||||
sslCtx = (SSLContext)sslCtxObj;
|
||||
}else if(sslCtxObj instanceof CharSequence){
|
||||
//assume this is a groovy code to get ssl context
|
||||
sslCtx = evaluateSSLContext((CharSequence)sslCtxObj);
|
||||
}else{
|
||||
throw new IllegalArgumentException("Unsupported ssl parameter ${sslCtxObj.getClass()}")
|
||||
}
|
||||
((HttpsURLConnection)connection).setSSLSocketFactory(sslCtx.getSocketFactory());
|
||||
}
|
||||
|
||||
connection.setDoOutput(true);
|
||||
connection.setRequestMethod(method);
|
||||
if ( headers!=null && !headers.isEmpty() ) {
|
||||
//add headers
|
||||
for (Map.Entry<String, String> entry : headers.entrySet()) {
|
||||
connection.addRequestProperty(entry.getKey(), entry.getValue());
|
||||
if("content-type".equals(entry.getKey().toLowerCase()))contentType=entry.getValue();
|
||||
if(entry.getValue()){
|
||||
connection.addRequestProperty(entry.getKey(), entry.getValue());
|
||||
if("content-type".equals(entry.getKey().toLowerCase()))contentType=entry.getValue();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -97,18 +137,20 @@ public class HTTP{
|
|||
}else if(body instanceof InputStream){
|
||||
out << (InputStream)body;
|
||||
}else if(body instanceof Map){
|
||||
if( contentType.matches("(?i)[^/]+/json") ){
|
||||
if( contentType =~ "(?i)[^/]+/json" ) {
|
||||
out.withWriter((String)ctx.encoding){
|
||||
it.append( JsonOutput.toJson((Map)body) );
|
||||
it.flush();
|
||||
}
|
||||
}else{
|
||||
throw new IOException("Map body type supported only for */json content-type");
|
||||
} else if( contentType =~ "(?i)[^/]+/x-www-form-urlencoded" ) {
|
||||
out.withWriter((String)ctx.encoding) {
|
||||
it.append( ((Map)body).collect{k,v-> ""+k+"="+URLEncoder.encode((String)v,'UTF-8') }.join('&') )
|
||||
}
|
||||
} else {
|
||||
throw new IOException("Map body type supported only for */json of */x-www-form-urlencoded content-type");
|
||||
}
|
||||
}else if(body instanceof CharSequence){
|
||||
out.withWriter((String)ctx.encoding){
|
||||
it.append((CharSequence)body);
|
||||
it.flush();
|
||||
}
|
||||
}else{
|
||||
throw new IOException("Unsupported body type: "+body.getClass());
|
||||
|
|
@ -151,4 +193,52 @@ public class HTTP{
|
|||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
@groovy.transform.Memoized
|
||||
public static SSLContext getKeystoreSSLContext(String keystorePath, String keystorePass, String keystoreType="JKS", String keyPass = null){
|
||||
if(keyPass == null) keyPass=keystorePass;
|
||||
KeyStore clientStore = KeyStore.getInstance(keystoreType);
|
||||
clientStore.load(new File( keystorePath ).newInputStream(), keystorePass.toCharArray());
|
||||
KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
|
||||
kmf.init(clientStore, keyPass.toCharArray());
|
||||
KeyManager[] kms = kmf.getKeyManagers();
|
||||
//init TrustCerts
|
||||
TrustManager[] trustCerts = new TrustManager[1];
|
||||
trustCerts[0] = new X509TrustManager() {
|
||||
public void checkClientTrusted( final X509Certificate[] chain, final String authType ) { }
|
||||
public void checkServerTrusted( final X509Certificate[] chain, final String authType ) { }
|
||||
public X509Certificate[] getAcceptedIssuers() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
SSLContext sslContext = SSLContext.getInstance("TLS");
|
||||
sslContext.init(kms, trustCerts, new SecureRandom());
|
||||
return sslContext;
|
||||
}
|
||||
|
||||
@groovy.transform.Memoized
|
||||
public static SSLContext getNaiveSSLContext(){
|
||||
System.err.println("HTTP.getNaiveSSLContext() used. Must be disabled on prod!");
|
||||
KeyManager[] kms = new KeyManager[0];
|
||||
TrustManager[] trustCerts = new TrustManager[1];
|
||||
trustCerts[0] = new X509TrustManager() {
|
||||
public void checkClientTrusted( final X509Certificate[] chain, final String authType ) { }
|
||||
public void checkServerTrusted( final X509Certificate[] chain, final String authType ) { }
|
||||
public X509Certificate[] getAcceptedIssuers() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
SSLContext sslContext = SSLContext.getInstance("TLS");
|
||||
sslContext.init(null, trustCerts, new SecureRandom());
|
||||
return sslContext;
|
||||
}
|
||||
|
||||
/**
|
||||
* evaluates code that should return SSLContext
|
||||
*/
|
||||
@groovy.transform.Memoized
|
||||
public static SSLContext evaluateSSLContext(CharSequence code) {
|
||||
Object ssl = new GroovyShell( HTTP.class.getClassLoader() ).evaluate( code as String );
|
||||
return (SSLContext) ssl;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -332,9 +332,6 @@ public class JDBCInterpreter extends Interpreter {
|
|||
|
||||
if (!getJDBCConfiguration(user).isConnectionInDBDriverPool(propertyKey)) {
|
||||
createConnectionPool(url, user, propertyKey, properties);
|
||||
try (Connection connection = DriverManager.getConnection(jdbcDriver)) {
|
||||
executePrecode(connection, propertyKey);
|
||||
}
|
||||
}
|
||||
return DriverManager.getConnection(jdbcDriver);
|
||||
}
|
||||
|
|
@ -572,18 +569,19 @@ public class JDBCInterpreter extends Interpreter {
|
|||
return queries;
|
||||
}
|
||||
|
||||
private void executePrecode(Connection connection, String propertyKey) throws SQLException {
|
||||
String precode = getProperty(String.format(PRECODE_KEY_TEMPLATE, propertyKey));
|
||||
if (StringUtils.isNotBlank(precode)) {
|
||||
precode = StringUtils.trim(precode);
|
||||
logger.debug("Run SQL precode '{}'", precode);
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.execute(precode);
|
||||
if (!connection.getAutoCommit()) {
|
||||
connection.commit();
|
||||
public InterpreterResult executePrecode(InterpreterContext interpreterContext) {
|
||||
InterpreterResult interpreterResult = null;
|
||||
for (String propertyKey : basePropretiesMap.keySet()) {
|
||||
String precode = getProperty(String.format("%s.precode", propertyKey));
|
||||
if (StringUtils.isNotBlank(precode)) {
|
||||
interpreterResult = executeSql(propertyKey, precode, interpreterContext);
|
||||
if (interpreterResult.code() != Code.SUCCESS) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return interpreterResult;
|
||||
}
|
||||
|
||||
private InterpreterResult executeSql(String propertyKey, String sql,
|
||||
|
|
|
|||
|
|
@ -400,17 +400,18 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
properties.setProperty("default.url", getJdbcConnection());
|
||||
properties.setProperty("default.user", "");
|
||||
properties.setProperty("default.password", "");
|
||||
properties.setProperty(DEFAULT_PRECODE, "SET @testVariable=1");
|
||||
properties.setProperty(DEFAULT_PRECODE, "create table test_precode (id int); insert into test_precode values (1);");
|
||||
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
|
||||
jdbcInterpreter.open();
|
||||
jdbcInterpreter.executePrecode(interpreterContext);
|
||||
|
||||
String sqlQuery = "select @testVariable";
|
||||
String sqlQuery = "select *from test_precode";
|
||||
|
||||
InterpreterResult interpreterResult = jdbcInterpreter.interpret(sqlQuery, interpreterContext);
|
||||
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
|
||||
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
|
||||
assertEquals("@TESTVARIABLE\n1\n", interpreterResult.message().get(0).getData());
|
||||
assertEquals("ID\n1\n", interpreterResult.message().get(0).getData());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -420,13 +421,15 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
properties.setProperty("default.url", getJdbcConnection());
|
||||
properties.setProperty("default.user", "");
|
||||
properties.setProperty("default.password", "");
|
||||
properties.setProperty(DEFAULT_PRECODE, "incorrect command");
|
||||
properties.setProperty(DEFAULT_PRECODE, "select 1");
|
||||
properties.setProperty("incorrect.driver", "org.h2.Driver");
|
||||
properties.setProperty("incorrect.url", getJdbcConnection());
|
||||
properties.setProperty("incorrect.user", "");
|
||||
properties.setProperty("incorrect.password", "");
|
||||
properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "incorrect"), "incorrect command");
|
||||
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
|
||||
jdbcInterpreter.open();
|
||||
|
||||
String sqlQuery = "select 1";
|
||||
|
||||
InterpreterResult interpreterResult = jdbcInterpreter.interpret(sqlQuery, interpreterContext);
|
||||
InterpreterResult interpreterResult = jdbcInterpreter.executePrecode(interpreterContext);
|
||||
|
||||
assertEquals(InterpreterResult.Code.ERROR, interpreterResult.code());
|
||||
assertEquals(InterpreterResult.Type.TEXT, interpreterResult.message().get(0).getType());
|
||||
|
|
@ -439,17 +442,18 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter {
|
|||
properties.setProperty("anotherPrefix.url", getJdbcConnection());
|
||||
properties.setProperty("anotherPrefix.user", "");
|
||||
properties.setProperty("anotherPrefix.password", "");
|
||||
properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"), "SET @testVariable=2");
|
||||
properties.setProperty(String.format(PRECODE_KEY_TEMPLATE, "anotherPrefix"), "create table test_precode_2 (id int); insert into test_precode_2 values (2);");
|
||||
JDBCInterpreter jdbcInterpreter = new JDBCInterpreter(properties);
|
||||
jdbcInterpreter.open();
|
||||
jdbcInterpreter.executePrecode(interpreterContext);
|
||||
|
||||
String sqlQuery = "(anotherPrefix) select @testVariable";
|
||||
String sqlQuery = "(anotherPrefix) select *from test_precode_2";
|
||||
|
||||
InterpreterResult interpreterResult = jdbcInterpreter.interpret(sqlQuery, interpreterContext);
|
||||
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
|
||||
assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType());
|
||||
assertEquals("@TESTVARIABLE\n2\n", interpreterResult.message().get(0).getData());
|
||||
assertEquals("ID\n2\n", interpreterResult.message().get(0).getData());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -555,13 +555,17 @@ public class LivyInterpreterIT {
|
|||
+ "df.collect()", context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertEquals(1, result.message().size());
|
||||
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]"));
|
||||
//python2 has u and python3 don't have u
|
||||
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]")
|
||||
|| result.message().get(0).getData().contains("[Row(_1='hello', _2=20)]"));
|
||||
} else {
|
||||
result = pysparkInterpreter.interpret("df=spark.createDataFrame([(\"hello\",20)])\n"
|
||||
+ "df.collect()", context);
|
||||
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
|
||||
assertEquals(1, result.message().size());
|
||||
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]"));
|
||||
//python2 has u and python3 don't have u
|
||||
assertTrue(result.message().get(0).getData().contains("[Row(_1=u'hello', _2=20)]")
|
||||
|| result.message().get(0).getData().contains("[Row(_1='hello', _2=20)]"));
|
||||
}
|
||||
|
||||
// test magic api
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ package org.apache.zeppelin.rinterpreter;
|
|||
import org.apache.spark.SparkContext;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
import org.apache.spark.sql.SQLContext;
|
||||
import org.apache.zeppelin.spark.ZeppelinContext;
|
||||
import org.apache.zeppelin.spark.SparkZeppelinContext;
|
||||
|
||||
/**
|
||||
* RStatics provides static class methods that can be accessed through the SparkR bridge
|
||||
|
|
@ -33,7 +33,7 @@ import org.apache.zeppelin.spark.ZeppelinContext;
|
|||
*/
|
||||
public class RStatics {
|
||||
private static SparkContext sc = null;
|
||||
private static ZeppelinContext z = null;
|
||||
private static SparkZeppelinContext z = null;
|
||||
private static SQLContext sql = null;
|
||||
private static RContext rCon = null;
|
||||
|
||||
|
|
@ -42,7 +42,7 @@ public class RStatics {
|
|||
return sc;
|
||||
}
|
||||
|
||||
public static ZeppelinContext setZ(ZeppelinContext newZ) {
|
||||
public static SparkZeppelinContext setZ(SparkZeppelinContext newZ) {
|
||||
z = newZ;
|
||||
return z;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import org.apache.zeppelin.interpreter._
|
|||
import org.apache.zeppelin.rinterpreter.rscala.RClient._
|
||||
import org.apache.zeppelin.rinterpreter.rscala._
|
||||
import org.apache.zeppelin.scheduler._
|
||||
import org.apache.zeppelin.spark.{SparkInterpreter, ZeppelinContext}
|
||||
import org.apache.zeppelin.spark.{SparkInterpreter, SparkZeppelinContext}
|
||||
import org.slf4j._
|
||||
|
||||
import scala.collection.JavaConversions._
|
||||
|
|
@ -45,7 +45,7 @@ private[rinterpreter] class RContext(private val sockets: ScalaSockets,
|
|||
val backend: RBackendHelper = RBackendHelper()
|
||||
private var sc: Option[SparkContext] = None
|
||||
private var sql: Option[SQLContext] = None
|
||||
private var z: Option[ZeppelinContext] = None
|
||||
private var z: Option[SparkZeppelinContext] = None
|
||||
|
||||
val rPkgMatrix = collection.mutable.HashMap[String,Boolean]()
|
||||
|
||||
|
|
@ -126,7 +126,7 @@ private[rinterpreter] class RContext(private val sockets: ScalaSockets,
|
|||
check whether SPARK_HOME is set properly.""", e)
|
||||
}
|
||||
|
||||
private def initializeSparkR(sc : SparkContext, sql : SQLContext, z : ZeppelinContext) : Unit = synchronized {
|
||||
private def initializeSparkR(sc : SparkContext, sql : SQLContext, z : SparkZeppelinContext) : Unit = synchronized {
|
||||
|
||||
logger.trace("Getting a handle to the JavaSparkContext")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
FROM alpine:3.4
|
||||
MAINTAINER Apache Software Foundation <dev@zeppelin.apache.org>
|
||||
|
||||
ENV JAVA_HOME /usr/lib/jvm/java-1.7-openjdk
|
||||
ENV PATH $PATH:$JAVA_HOME/bin
|
||||
|
||||
RUN apk add --update bash curl openjdk7-jre wget ca-certificates python build-base make gcc g++ java-cacerts openssl && \
|
||||
rm /usr/lib/jvm/java-1.7-openjdk/jre/lib/security/cacerts && \
|
||||
ln -s /etc/ssl/certs/java/cacerts /usr/lib/jvm/java-1.7-openjdk/jre/lib/security/cacerts && \
|
||||
curl --silent \
|
||||
--location https://github.com/sgerrand/alpine-pkg-R/releases/download/3.3.1-r0/R-3.3.1-r0.apk --output /var/cache/apk/R-3.3.1-r0.apk && \
|
||||
apk add --update --allow-untrusted /var/cache/apk/R-3.3.1-r0.apk && \
|
||||
curl --silent \
|
||||
--location https://github.com/sgerrand/alpine-pkg-R/releases/download/3.3.1-r0/R-dev-3.3.1-r0.apk --output /var/cache/apk/R-dev-3.3.1-r0.apk && \
|
||||
apk add --update --allow-untrusted /var/cache/apk/R-dev-3.3.1-r0.apk && \
|
||||
R -e "install.packages('knitr', repos = 'http://cran.us.r-project.org')" && \
|
||||
apk del curl build-base make gcc g++ && \
|
||||
rm -rf /var/cache/apk/*
|
||||
|
||||
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64
|
||||
RUN chmod +x /usr/local/bin/dumb-init
|
||||
|
||||
# ports for zeppelin
|
||||
EXPOSE 8080 7077
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
|
||||
37
scripts/docker/zeppelin/0.6.2/Dockerfile
Normal file
37
scripts/docker/zeppelin/0.6.2/Dockerfile
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM zeppelin:base
|
||||
MAINTAINER Apache Software Foundation <dev@zeppelin.apache.org>
|
||||
|
||||
ENV Z_VERSION="0.6.2"
|
||||
ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \
|
||||
Z_HOME="/zeppelin"
|
||||
|
||||
RUN echo "$LOG_TAG Download Zeppelin binary" && \
|
||||
wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME}
|
||||
|
||||
RUN echo "$LOG_TAG Cleanup" && \
|
||||
apt-get autoclean && \
|
||||
apt-get clean
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
WORKDIR ${Z_HOME}
|
||||
CMD ["bin/zeppelin.sh"]
|
||||
|
||||
37
scripts/docker/zeppelin/0.7.0/Dockerfile
Normal file
37
scripts/docker/zeppelin/0.7.0/Dockerfile
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM zeppelin:base
|
||||
MAINTAINER Apache Software Foundation <dev@zeppelin.apache.org>
|
||||
|
||||
ENV Z_VERSION="0.7.0"
|
||||
ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \
|
||||
Z_HOME="/zeppelin"
|
||||
|
||||
RUN echo "$LOG_TAG Download Zeppelin binary" && \
|
||||
wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
rm -rf /zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME}
|
||||
|
||||
RUN echo "$LOG_TAG Cleanup" && \
|
||||
apt-get autoclean && \
|
||||
apt-get clean
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
WORKDIR ${Z_HOME}
|
||||
CMD ["bin/zeppelin.sh"]
|
||||
|
||||
37
scripts/docker/zeppelin/0.7.1/Dockerfile
Normal file
37
scripts/docker/zeppelin/0.7.1/Dockerfile
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM zeppelin:base
|
||||
MAINTAINER Apache Software Foundation <dev@zeppelin.apache.org>
|
||||
|
||||
ENV Z_VERSION="0.7.1"
|
||||
ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \
|
||||
Z_HOME="/zeppelin"
|
||||
|
||||
RUN echo "$LOG_TAG Download Zeppelin binary" && \
|
||||
wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME}
|
||||
|
||||
RUN echo "$LOG_TAG Cleanup" && \
|
||||
apt-get autoclean && \
|
||||
apt-get clean
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
WORKDIR ${Z_HOME}
|
||||
CMD ["bin/zeppelin.sh"]
|
||||
|
||||
90
scripts/docker/zeppelin/base/Dockerfile
Normal file
90
scripts/docker/zeppelin/base/Dockerfile
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM ubuntu:16.04
|
||||
MAINTAINER Apache Software Foundation <dev@zeppelin.apache.org>
|
||||
|
||||
ENV LOG_TAG="[ZEPPELIN_BASE]:" \
|
||||
LANG=en_US.UTF-8 \
|
||||
LC_ALL=en_US.UTF-8
|
||||
|
||||
RUN echo "$LOG_TAG update and install basic packages" && \
|
||||
apt-get -y update && \
|
||||
apt-get install -y locales && \
|
||||
locale-gen $LANG && \
|
||||
apt-get install -y software-properties-common && \
|
||||
apt -y autoclean && \
|
||||
apt -y dist-upgrade && \
|
||||
apt-get install -y build-essential
|
||||
|
||||
RUN echo "$LOG_TAG install tini related packages" && \
|
||||
apt-get install -y curl grep sed dpkg && \
|
||||
TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \
|
||||
curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \
|
||||
dpkg -i tini.deb && \
|
||||
rm tini.deb
|
||||
|
||||
ENV JAVA_HOME=/usr/lib/jvm/java-8-oracle
|
||||
RUN echo "$LOG_TAG Install java8" && \
|
||||
echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
|
||||
add-apt-repository -y ppa:webupd8team/java && \
|
||||
apt-get -y update && \
|
||||
apt-get install -y oracle-java8-installer && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
rm -rf /var/cache/oracle-jdk8-installer
|
||||
|
||||
# should install conda first before numpy, matploylib since pip and python will be installed by conda
|
||||
RUN echo "$LOG_TAG Install miniconda2 related packages" && \
|
||||
apt-get -y update && \
|
||||
apt-get install -y bzip2 ca-certificates \
|
||||
libglib2.0-0 libxext6 libsm6 libxrender1 \
|
||||
git mercurial subversion && \
|
||||
echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \
|
||||
wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.11-Linux-x86_64.sh -O ~/miniconda.sh && \
|
||||
/bin/bash ~/miniconda.sh -b -p /opt/conda && \
|
||||
rm ~/miniconda.sh
|
||||
ENV PATH /opt/conda/bin:$PATH
|
||||
|
||||
RUN echo "$LOG_TAG Install python related packages" && \
|
||||
apt-get -y update && \
|
||||
apt-get install -y python-dev python-pip && \
|
||||
apt-get install -y gfortran && \
|
||||
# numerical/algebra packages
|
||||
apt-get install -y libblas-dev libatlas-dev liblapack-dev && \
|
||||
# font, image for matplotlib
|
||||
apt-get install -y libpng-dev libfreetype6-dev libxft-dev && \
|
||||
# for tkinter
|
||||
apt-get install -y python-tk libxml2-dev libxslt-dev zlib1g-dev && \
|
||||
pip install numpy && \
|
||||
pip install matplotlib
|
||||
|
||||
RUN echo "$LOG_TAG Install R related packages" && \
|
||||
echo "deb http://cran.rstudio.com/bin/linux/ubuntu xenial/" | tee -a /etc/apt/sources.list && \
|
||||
gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 && \
|
||||
gpg -a --export E084DAB9 | apt-key add - && \
|
||||
apt-get -y update && \
|
||||
apt-get -y install r-base r-base-dev && \
|
||||
R -e "install.packages('knitr', repos='http://cran.us.r-project.org')" && \
|
||||
R -e "install.packages('ggplot2', repos='http://cran.us.r-project.org')" && \
|
||||
R -e "install.packages('googleVis', repos='http://cran.us.r-project.org')" && \
|
||||
R -e "install.packages('data.table', repos='http://cran.us.r-project.org')" && \
|
||||
# for devtools, Rcpp
|
||||
apt-get -y install libcurl4-gnutls-dev libssl-dev && \
|
||||
R -e "install.packages('devtools', repos='http://cran.us.r-project.org')" && \
|
||||
R -e "install.packages('Rcpp', repos='http://cran.us.r-project.org')" && \
|
||||
Rscript -e "library('devtools'); library('Rcpp'); install_github('ramnathv/rCharts')"
|
||||
|
||||
ENTRYPOINT [ "/usr/bin/tini", "--" ]
|
||||
CMD [ "/bin/bash" ]
|
||||
37
scripts/docker/zeppelin/bin-template/Dockerfile
Normal file
37
scripts/docker/zeppelin/bin-template/Dockerfile
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM zeppelin:base
|
||||
MAINTAINER Apache Software Foundation <dev@zeppelin.apache.org>
|
||||
|
||||
ENV Z_VERSION="0.0.0"
|
||||
ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \
|
||||
Z_HOME="/zeppelin"
|
||||
|
||||
RUN echo "$LOG_TAG Download Zeppelin binary" && \
|
||||
wget -O /tmp/zeppelin-${Z_VERSION}-bin-all.tgz http://archive.apache.org/dist/zeppelin/zeppelin-${Z_VERSION}/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
tar -zxvf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
rm -rf /tmp/zeppelin-${Z_VERSION}-bin-all.tgz && \
|
||||
mv /zeppelin-${Z_VERSION}-bin-all ${Z_HOME}
|
||||
|
||||
RUN echo "$LOG_TAG Cleanup" && \
|
||||
apt-get autoclean && \
|
||||
apt-get clean
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
WORKDIR ${Z_HOME}
|
||||
CMD ["bin/zeppelin.sh"]
|
||||
|
||||
51
scripts/docker/zeppelin/create-dockerfile.sh
Executable file
51
scripts/docker/zeppelin/create-dockerfile.sh
Executable file
|
|
@ -0,0 +1,51 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
if [ $# -lt 1 ];
|
||||
then
|
||||
echo "USAGE: $0 version"
|
||||
echo "* version: 0.6.2 (released zeppelin binary version)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TAG="[CREATE-DOCKERFILE]"
|
||||
VERSION=$1
|
||||
|
||||
BASE_DIR="./base/"
|
||||
TEMPLATE_DOCKERFILE="./bin-template/Dockerfile"
|
||||
|
||||
if [ ! -d "$BASE_DIR" ]; then
|
||||
echo "${TAG} Base Directory doesn't exist: ${BASE_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET_DIR="${VERSION}"
|
||||
BASE_IMAGE_TAG="base"
|
||||
|
||||
if [ ! -d "$TARGET_DIR" ]; then
|
||||
echo "${TAG} Creating Directory: ${TARGET_DIR}"
|
||||
mkdir -p ${TARGET_DIR}
|
||||
|
||||
echo "${TAG} Copying File: ${TARGET_DIR}/Dockerfile"
|
||||
cp ${TEMPLATE_DOCKERFILE} ${TARGET_DIR}/Dockerfile
|
||||
|
||||
echo "${TAG} Set Version: ${VERSION}"
|
||||
sed -i '' -e "s/Z_VERSION=\"0.0.0\"/Z_VERSION=\"${VERSION}\"/g" ${TARGET_DIR}/Dockerfile
|
||||
else
|
||||
echo "${TAG} Directory already exists: ${TARGET_DIR}"
|
||||
fi
|
||||
|
|
@ -63,7 +63,7 @@ import py4j.GatewayServer;
|
|||
*
|
||||
*/
|
||||
public class PySparkInterpreter extends Interpreter implements ExecuteResultHandler {
|
||||
Logger logger = LoggerFactory.getLogger(PySparkInterpreter.class);
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PySparkInterpreter.class);
|
||||
private GatewayServer gatewayServer;
|
||||
private DefaultExecutor executor;
|
||||
private int port;
|
||||
|
|
@ -106,7 +106,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
throw new InterpreterException(e);
|
||||
}
|
||||
|
||||
logger.info("File {} created", scriptPath);
|
||||
LOGGER.info("File {} created", scriptPath);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -131,7 +131,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
try {
|
||||
urlList.add(f.toURI().toURL());
|
||||
} catch (MalformedURLException e) {
|
||||
logger.error("Error", e);
|
||||
LOGGER.error("Error", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -148,7 +148,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
try {
|
||||
urlList.add(f.toURI().toURL());
|
||||
} catch (MalformedURLException e) {
|
||||
logger.error("Error", e);
|
||||
LOGGER.error("Error", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -162,7 +162,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
Thread.currentThread().setContextClassLoader(newCl);
|
||||
createGatewayServerAndStartScript();
|
||||
} catch (Exception e) {
|
||||
logger.error("Error", e);
|
||||
LOGGER.error("Error", e);
|
||||
throw new InterpreterException(e);
|
||||
} finally {
|
||||
Thread.currentThread().setContextClassLoader(oldCl);
|
||||
|
|
@ -217,7 +217,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
cmd.addArgument(Integer.toString(port), false);
|
||||
cmd.addArgument(Integer.toString(getSparkInterpreter().getSparkVersion().toNumber()), false);
|
||||
executor = new DefaultExecutor();
|
||||
outputStream = new InterpreterOutputStream(logger);
|
||||
outputStream = new InterpreterOutputStream(LOGGER);
|
||||
PipedOutputStream ps = new PipedOutputStream();
|
||||
in = null;
|
||||
try {
|
||||
|
|
@ -313,6 +313,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
|
||||
public void setStatementsFinished(String out, boolean error) {
|
||||
synchronized (statementFinishedNotifier) {
|
||||
LOGGER.debug("Setting python statement output: " + out + ", error: " + error);
|
||||
statementOutput = out;
|
||||
statementError = error;
|
||||
statementFinishedNotifier.notify();
|
||||
|
|
@ -325,12 +326,14 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
public void onPythonScriptInitialized(long pid) {
|
||||
pythonPid = pid;
|
||||
synchronized (pythonScriptInitializeNotifier) {
|
||||
LOGGER.debug("onPythonScriptInitialized is called");
|
||||
pythonScriptInitialized = true;
|
||||
pythonScriptInitializeNotifier.notifyAll();
|
||||
}
|
||||
}
|
||||
|
||||
public void appendOutput(String message) throws IOException {
|
||||
LOGGER.debug("Output from python process: " + message);
|
||||
outputStream.getInterpreterOutput().write(message);
|
||||
}
|
||||
|
||||
|
|
@ -358,6 +361,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
try {
|
||||
pythonScriptInitializeNotifier.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -391,7 +395,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
return new InterpreterResult(Code.ERROR, errorMessage);
|
||||
}
|
||||
String jobGroup = Utils.buildJobGroupId(context);
|
||||
ZeppelinContext __zeppelin__ = sparkInterpreter.getZeppelinContext();
|
||||
SparkZeppelinContext __zeppelin__ = sparkInterpreter.getZeppelinContext();
|
||||
__zeppelin__.setInterpreterContext(context);
|
||||
__zeppelin__.setGui(context.getGui());
|
||||
pythonInterpretRequest = new PythonInterpretRequest(st, jobGroup);
|
||||
|
|
@ -426,10 +430,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
|
||||
public void interrupt() throws IOException {
|
||||
if (pythonPid > -1) {
|
||||
logger.info("Sending SIGINT signal to PID : " + pythonPid);
|
||||
LOGGER.info("Sending SIGINT signal to PID : " + pythonPid);
|
||||
Runtime.getRuntime().exec("kill -SIGINT " + pythonPid);
|
||||
} else {
|
||||
logger.warn("Non UNIX/Linux system, close the interpreter");
|
||||
LOGGER.warn("Non UNIX/Linux system, close the interpreter");
|
||||
close();
|
||||
}
|
||||
}
|
||||
|
|
@ -441,7 +445,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
try {
|
||||
interrupt();
|
||||
} catch (IOException e) {
|
||||
logger.error("Error", e);
|
||||
LOGGER.error("Error", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -486,13 +490,13 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
&& pythonscriptRunning) {
|
||||
try {
|
||||
if (System.currentTimeMillis() - startTime > MAX_TIMEOUT_SEC * 1000) {
|
||||
logger.error("pyspark completion didn't have response for {}sec.", MAX_TIMEOUT_SEC);
|
||||
LOGGER.error("pyspark completion didn't have response for {}sec.", MAX_TIMEOUT_SEC);
|
||||
break;
|
||||
}
|
||||
statementFinishedNotifier.wait(1000);
|
||||
} catch (InterruptedException e) {
|
||||
// not working
|
||||
logger.info("wait drop");
|
||||
LOGGER.info("wait drop");
|
||||
return new LinkedList<>();
|
||||
}
|
||||
}
|
||||
|
|
@ -527,7 +531,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
completionScriptText = text.substring(0, cursor);
|
||||
}
|
||||
catch (Exception e) {
|
||||
logger.error(e.toString());
|
||||
LOGGER.error(e.toString());
|
||||
return null;
|
||||
}
|
||||
completionEndPosition = completionScriptText.length();
|
||||
|
|
@ -576,7 +580,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
return spark;
|
||||
}
|
||||
|
||||
public ZeppelinContext getZeppelinContext() {
|
||||
public SparkZeppelinContext getZeppelinContext() {
|
||||
SparkInterpreter sparkIntp = getSparkInterpreter();
|
||||
if (sparkIntp != null) {
|
||||
return getSparkInterpreter().getZeppelinContext();
|
||||
|
|
@ -637,12 +641,12 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
|
|||
@Override
|
||||
public void onProcessComplete(int exitValue) {
|
||||
pythonscriptRunning = false;
|
||||
logger.info("python process terminated. exit code " + exitValue);
|
||||
LOGGER.info("python process terminated. exit code " + exitValue);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onProcessFailed(ExecuteException e) {
|
||||
pythonscriptRunning = false;
|
||||
logger.error("python process failed", e);
|
||||
LOGGER.error("python process failed", e);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ import org.apache.spark.scheduler.SparkListenerJobStart;
|
|||
import org.apache.spark.sql.SQLContext;
|
||||
import org.apache.spark.ui.SparkUI;
|
||||
import org.apache.spark.ui.jobs.JobProgressListener;
|
||||
import org.apache.zeppelin.interpreter.BaseZeppelinContext;
|
||||
import org.apache.zeppelin.interpreter.DefaultInterpreterProperty;
|
||||
import org.apache.zeppelin.interpreter.Interpreter;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
|
|
@ -103,7 +104,7 @@ import scala.tools.nsc.settings.MutableSettings.PathSetting;
|
|||
public class SparkInterpreter extends Interpreter {
|
||||
public static Logger logger = LoggerFactory.getLogger(SparkInterpreter.class);
|
||||
|
||||
private ZeppelinContext z;
|
||||
private SparkZeppelinContext z;
|
||||
private SparkILoop interpreter;
|
||||
/**
|
||||
* intp - org.apache.spark.repl.SparkIMain (scala 2.10)
|
||||
|
|
@ -187,7 +188,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
String noteId = Utils.getNoteId(jobGroupId);
|
||||
String paragraphId = Utils.getParagraphId(jobGroupId);
|
||||
if (jobUrl != null && noteId != null && paragraphId != null) {
|
||||
RemoteEventClientWrapper eventClient = ZeppelinContext.getEventClient();
|
||||
RemoteEventClientWrapper eventClient = BaseZeppelinContext.getEventClient();
|
||||
Map<String, String> infos = new java.util.HashMap<>();
|
||||
infos.put("jobUrl", jobUrl);
|
||||
infos.put("label", "SPARK JOB");
|
||||
|
|
@ -888,7 +889,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
|
||||
hooks = getInterpreterGroup().getInterpreterHookRegistry();
|
||||
|
||||
z = new ZeppelinContext(sc, sqlc, null, dep, hooks,
|
||||
z = new SparkZeppelinContext(sc, sqlc, hooks,
|
||||
Integer.parseInt(getProperty("zeppelin.spark.maxResult")));
|
||||
|
||||
interpret("@transient val _binder = new java.util.HashMap[String, Object]()");
|
||||
|
|
@ -907,7 +908,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
}
|
||||
|
||||
interpret("@transient val z = "
|
||||
+ "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]");
|
||||
+ "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.SparkZeppelinContext]");
|
||||
interpret("@transient val sc = "
|
||||
+ "_binder.get(\"sc\").asInstanceOf[org.apache.spark.SparkContext]");
|
||||
interpret("@transient val sqlc = "
|
||||
|
|
@ -1486,7 +1487,7 @@ public class SparkInterpreter extends Interpreter {
|
|||
SparkInterpreter.class.getName() + this.hashCode());
|
||||
}
|
||||
|
||||
public ZeppelinContext getZeppelinContext() {
|
||||
public SparkZeppelinContext getZeppelinContext() {
|
||||
return z;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ public class SparkSqlInterpreter extends Interpreter {
|
|||
throw new InterpreterException(e);
|
||||
}
|
||||
|
||||
String msg = ZeppelinContext.showDF(sc, context, rdd, maxResult);
|
||||
String msg = getSparkInterpreter().getZeppelinContext().showData(rdd);
|
||||
sc.clearJobGroup();
|
||||
return new InterpreterResult(Code.SUCCESS, msg);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,280 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.spark;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.spark.SparkContext;
|
||||
import org.apache.spark.sql.SQLContext;
|
||||
import org.apache.spark.sql.catalyst.expressions.Attribute;
|
||||
import org.apache.zeppelin.annotation.ZeppelinApi;
|
||||
import org.apache.zeppelin.display.AngularObjectWatcher;
|
||||
import org.apache.zeppelin.display.Input;
|
||||
import org.apache.zeppelin.display.ui.OptionInput;
|
||||
import org.apache.zeppelin.interpreter.*;
|
||||
import scala.Tuple2;
|
||||
import scala.Unit;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.*;
|
||||
|
||||
import static scala.collection.JavaConversions.asJavaCollection;
|
||||
import static scala.collection.JavaConversions.asJavaIterable;
|
||||
import static scala.collection.JavaConversions.collectionAsScalaIterable;
|
||||
|
||||
/**
|
||||
* ZeppelinContext for Spark
|
||||
*/
|
||||
public class SparkZeppelinContext extends BaseZeppelinContext {
|
||||
|
||||
|
||||
private SparkContext sc;
|
||||
public SQLContext sqlContext;
|
||||
private List<Class> supportedClasses;
|
||||
private Map<String, String> interpreterClassMap;
|
||||
|
||||
public SparkZeppelinContext(
|
||||
SparkContext sc, SQLContext sql,
|
||||
InterpreterHookRegistry hooks,
|
||||
int maxResult) {
|
||||
super(hooks, maxResult);
|
||||
this.sc = sc;
|
||||
this.sqlContext = sql;
|
||||
|
||||
interpreterClassMap = new HashMap<String, String>();
|
||||
interpreterClassMap.put("spark", "org.apache.zeppelin.spark.SparkInterpreter");
|
||||
interpreterClassMap.put("sql", "org.apache.zeppelin.spark.SparkSqlInterpreter");
|
||||
interpreterClassMap.put("dep", "org.apache.zeppelin.spark.DepInterpreter");
|
||||
interpreterClassMap.put("pyspark", "org.apache.zeppelin.spark.PySparkInterpreter");
|
||||
|
||||
this.supportedClasses = new ArrayList<>();
|
||||
try {
|
||||
supportedClasses.add(this.getClass().forName("org.apache.spark.sql.Dataset"));
|
||||
} catch (ClassNotFoundException e) {
|
||||
}
|
||||
|
||||
try {
|
||||
supportedClasses.add(this.getClass().forName("org.apache.spark.sql.DataFrame"));
|
||||
} catch (ClassNotFoundException e) {
|
||||
}
|
||||
|
||||
try {
|
||||
supportedClasses.add(this.getClass().forName("org.apache.spark.sql.SchemaRDD"));
|
||||
} catch (ClassNotFoundException e) {
|
||||
}
|
||||
|
||||
if (supportedClasses.isEmpty()) {
|
||||
throw new InterpreterException("Can not load Dataset/DataFrame/SchemaRDD class");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Class> getSupportedClasses() {
|
||||
return supportedClasses;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> getInterpreterClassMap() {
|
||||
return interpreterClassMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String showData(Object df) {
|
||||
Object[] rows = null;
|
||||
Method take;
|
||||
String jobGroup = Utils.buildJobGroupId(interpreterContext);
|
||||
sc.setJobGroup(jobGroup, "Zeppelin", false);
|
||||
|
||||
try {
|
||||
// convert it to DataFrame if it is Dataset, as we will iterate all the records
|
||||
// and assume it is type Row.
|
||||
if (df.getClass().getCanonicalName().equals("org.apache.spark.sql.Dataset")) {
|
||||
Method convertToDFMethod = df.getClass().getMethod("toDF");
|
||||
df = convertToDFMethod.invoke(df);
|
||||
}
|
||||
take = df.getClass().getMethod("take", int.class);
|
||||
rows = (Object[]) take.invoke(df, maxResult + 1);
|
||||
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
|
||||
| IllegalArgumentException | InvocationTargetException | ClassCastException e) {
|
||||
sc.clearJobGroup();
|
||||
throw new InterpreterException(e);
|
||||
}
|
||||
|
||||
List<Attribute> columns = null;
|
||||
// get field names
|
||||
try {
|
||||
// Use reflection because of classname returned by queryExecution changes from
|
||||
// Spark <1.5.2 org.apache.spark.sql.SQLContext$QueryExecution
|
||||
// Spark 1.6.0> org.apache.spark.sql.hive.HiveContext$QueryExecution
|
||||
Object qe = df.getClass().getMethod("queryExecution").invoke(df);
|
||||
Object a = qe.getClass().getMethod("analyzed").invoke(qe);
|
||||
scala.collection.Seq seq = (scala.collection.Seq) a.getClass().getMethod("output").invoke(a);
|
||||
|
||||
columns = (List<Attribute>) scala.collection.JavaConverters.seqAsJavaListConverter(seq)
|
||||
.asJava();
|
||||
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
|
||||
| IllegalArgumentException | InvocationTargetException e) {
|
||||
throw new InterpreterException(e);
|
||||
}
|
||||
|
||||
StringBuilder msg = new StringBuilder();
|
||||
msg.append("%table ");
|
||||
for (Attribute col : columns) {
|
||||
msg.append(col.name() + "\t");
|
||||
}
|
||||
String trim = msg.toString().trim();
|
||||
msg = new StringBuilder(trim);
|
||||
msg.append("\n");
|
||||
|
||||
// ArrayType, BinaryType, BooleanType, ByteType, DecimalType, DoubleType, DynamicType,
|
||||
// FloatType, FractionalType, IntegerType, IntegralType, LongType, MapType, NativeType,
|
||||
// NullType, NumericType, ShortType, StringType, StructType
|
||||
|
||||
try {
|
||||
for (int r = 0; r < maxResult && r < rows.length; r++) {
|
||||
Object row = rows[r];
|
||||
Method isNullAt = row.getClass().getMethod("isNullAt", int.class);
|
||||
Method apply = row.getClass().getMethod("apply", int.class);
|
||||
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
if (!(Boolean) isNullAt.invoke(row, i)) {
|
||||
msg.append(apply.invoke(row, i).toString());
|
||||
} else {
|
||||
msg.append("null");
|
||||
}
|
||||
if (i != columns.size() - 1) {
|
||||
msg.append("\t");
|
||||
}
|
||||
}
|
||||
msg.append("\n");
|
||||
}
|
||||
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
|
||||
| IllegalArgumentException | InvocationTargetException e) {
|
||||
throw new InterpreterException(e);
|
||||
}
|
||||
|
||||
if (rows.length > maxResult) {
|
||||
msg.append("\n");
|
||||
msg.append(ResultMessages.getExceedsLimitRowsMessage(maxResult,
|
||||
SparkSqlInterpreter.MAX_RESULTS));
|
||||
}
|
||||
|
||||
sc.clearJobGroup();
|
||||
return msg.toString();
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object select(String name, scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
return select(name, "", options);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object select(String name, Object defaultValue,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
return select(name, defaultValue, tuplesToParamOptions(options));
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public scala.collection.Seq<Object> checkbox(
|
||||
String name,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
List<Object> allChecked = new LinkedList<>();
|
||||
for (Tuple2<Object, String> option : asJavaIterable(options)) {
|
||||
allChecked.add(option._1());
|
||||
}
|
||||
return checkbox(name, collectionAsScalaIterable(allChecked), options);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public scala.collection.Seq<Object> checkbox(
|
||||
String name,
|
||||
scala.collection.Iterable<Object> defaultChecked,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
return scala.collection.JavaConversions.asScalaBuffer(
|
||||
gui.checkbox(name, asJavaCollection(defaultChecked),
|
||||
tuplesToParamOptions(options))).toSeq();
|
||||
}
|
||||
|
||||
private OptionInput.ParamOption[] tuplesToParamOptions(
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
int n = options.size();
|
||||
OptionInput.ParamOption[] paramOptions = new OptionInput.ParamOption[n];
|
||||
Iterator<Tuple2<Object, String>> it = asJavaIterable(options).iterator();
|
||||
|
||||
int i = 0;
|
||||
while (it.hasNext()) {
|
||||
Tuple2<Object, String> valueAndDisplayValue = it.next();
|
||||
paramOptions[i++] = new OptionInput.ParamOption(valueAndDisplayValue._1(),
|
||||
valueAndDisplayValue._2());
|
||||
}
|
||||
|
||||
return paramOptions;
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public void angularWatch(String name,
|
||||
final scala.Function2<Object, Object, Unit> func) {
|
||||
angularWatch(name, interpreterContext.getNoteId(), func);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void angularWatchGlobal(String name,
|
||||
final scala.Function2<Object, Object, Unit> func) {
|
||||
angularWatch(name, null, func);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public void angularWatch(
|
||||
String name,
|
||||
final scala.Function3<Object, Object, InterpreterContext, Unit> func) {
|
||||
angularWatch(name, interpreterContext.getNoteId(), func);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void angularWatchGlobal(
|
||||
String name,
|
||||
final scala.Function3<Object, Object, InterpreterContext, Unit> func) {
|
||||
angularWatch(name, null, func);
|
||||
}
|
||||
|
||||
private void angularWatch(String name, String noteId,
|
||||
final scala.Function2<Object, Object, Unit> func) {
|
||||
AngularObjectWatcher w = new AngularObjectWatcher(getInterpreterContext()) {
|
||||
@Override
|
||||
public void watch(Object oldObject, Object newObject,
|
||||
InterpreterContext context) {
|
||||
func.apply(newObject, newObject);
|
||||
}
|
||||
};
|
||||
angularWatch(name, noteId, w);
|
||||
}
|
||||
|
||||
private void angularWatch(
|
||||
String name,
|
||||
String noteId,
|
||||
final scala.Function3<Object, Object, InterpreterContext, Unit> func) {
|
||||
AngularObjectWatcher w = new AngularObjectWatcher(getInterpreterContext()) {
|
||||
@Override
|
||||
public void watch(Object oldObject, Object newObject,
|
||||
InterpreterContext context) {
|
||||
func.apply(oldObject, newObject, context);
|
||||
}
|
||||
};
|
||||
angularWatch(name, noteId, w);
|
||||
}
|
||||
}
|
||||
|
|
@ -27,7 +27,7 @@ import org.apache.spark.sql.SQLContext;
|
|||
public class ZeppelinRContext {
|
||||
private static SparkContext sparkContext;
|
||||
private static SQLContext sqlContext;
|
||||
private static ZeppelinContext zeppelinContext;
|
||||
private static SparkZeppelinContext zeppelinContext;
|
||||
private static Object sparkSession;
|
||||
private static JavaSparkContext javaSparkContext;
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ public class ZeppelinRContext {
|
|||
ZeppelinRContext.sparkContext = sparkContext;
|
||||
}
|
||||
|
||||
public static void setZeppelinContext(ZeppelinContext zeppelinContext) {
|
||||
public static void setZeppelinContext(SparkZeppelinContext zeppelinContext) {
|
||||
ZeppelinRContext.zeppelinContext = zeppelinContext;
|
||||
}
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ public class ZeppelinRContext {
|
|||
return sqlContext;
|
||||
}
|
||||
|
||||
public static ZeppelinContext getZeppelinContext() {
|
||||
public static SparkZeppelinContext getZeppelinContext() {
|
||||
return zeppelinContext;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,15 +21,7 @@ from py4j.java_gateway import java_import, JavaGateway, GatewayClient
|
|||
from py4j.protocol import Py4JJavaError
|
||||
from pyspark.conf import SparkConf
|
||||
from pyspark.context import SparkContext
|
||||
from pyspark.rdd import RDD
|
||||
from pyspark.files import SparkFiles
|
||||
from pyspark.storagelevel import StorageLevel
|
||||
from pyspark.accumulators import Accumulator, AccumulatorParam
|
||||
from pyspark.broadcast import Broadcast
|
||||
from pyspark.serializers import MarshalSerializer, PickleSerializer
|
||||
import warnings
|
||||
import ast
|
||||
import traceback
|
||||
import warnings
|
||||
|
||||
# for back compatibility
|
||||
|
|
@ -57,7 +49,7 @@ class PyZeppelinContext(dict):
|
|||
def show(self, obj):
|
||||
from pyspark.sql import DataFrame
|
||||
if isinstance(obj, DataFrame):
|
||||
print(gateway.jvm.org.apache.zeppelin.spark.ZeppelinContext.showDF(self.z, obj._jdf))
|
||||
print(self.z.showData(obj._jdf))
|
||||
else:
|
||||
print(str(obj))
|
||||
|
||||
|
|
@ -231,20 +223,13 @@ class PySparkCompletion:
|
|||
result = json.dumps(list(filter(lambda x : not re.match("^__.*", x), list(completionList))))
|
||||
self.interpreterObject.setStatementsFinished(result, False)
|
||||
|
||||
|
||||
output = Logger()
|
||||
sys.stdout = output
|
||||
sys.stderr = output
|
||||
|
||||
client = GatewayClient(port=int(sys.argv[1]))
|
||||
sparkVersion = SparkVersion(int(sys.argv[2]))
|
||||
|
||||
if sparkVersion.isSpark2():
|
||||
from pyspark.sql import SparkSession
|
||||
else:
|
||||
from pyspark.sql import SchemaRDD
|
||||
|
||||
|
||||
if sparkVersion.isAutoConvertEnabled():
|
||||
gateway = JavaGateway(client, auto_convert = True)
|
||||
else:
|
||||
|
|
@ -257,6 +242,9 @@ java_import(gateway.jvm, "org.apache.spark.api.python.*")
|
|||
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
|
||||
|
||||
intp = gateway.entry_point
|
||||
output = Logger()
|
||||
sys.stdout = output
|
||||
sys.stderr = output
|
||||
intp.onPythonScriptInitialized(os.getpid())
|
||||
|
||||
jsc = intp.getJavaSparkContext()
|
||||
|
|
@ -310,7 +298,6 @@ while True :
|
|||
try:
|
||||
stmts = req.statements().split("\n")
|
||||
jobGroup = req.jobGroup()
|
||||
final_code = []
|
||||
|
||||
# Get post-execute hooks
|
||||
try:
|
||||
|
|
@ -328,22 +315,11 @@ while True :
|
|||
if hook:
|
||||
nhooks += 1
|
||||
|
||||
for s in stmts:
|
||||
if s == None:
|
||||
continue
|
||||
|
||||
# skip comment
|
||||
s_stripped = s.strip()
|
||||
if len(s_stripped) == 0 or s_stripped.startswith("#"):
|
||||
continue
|
||||
|
||||
final_code.append(s)
|
||||
|
||||
if final_code:
|
||||
if stmts:
|
||||
# use exec mode to compile the statements except the last statement,
|
||||
# so that the last statement's evaluation will be printed to stdout
|
||||
sc.setJobGroup(jobGroup, "Zeppelin")
|
||||
code = compile('\n'.join(final_code), '<stdin>', 'exec', ast.PyCF_ONLY_AST, 1)
|
||||
code = compile('\n'.join(stmts), '<stdin>', 'exec', ast.PyCF_ONLY_AST, 1)
|
||||
to_run_hooks = []
|
||||
if (nhooks > 0):
|
||||
to_run_hooks = code.body[-nhooks:]
|
||||
|
|
@ -365,10 +341,23 @@ while True :
|
|||
mod = ast.Module([node])
|
||||
code = compile(mod, '<stdin>', 'exec')
|
||||
exec(code, _zcUserQueryNameSpace)
|
||||
except:
|
||||
raise Exception(traceback.format_exc())
|
||||
|
||||
intp.setStatementsFinished("", False)
|
||||
intp.setStatementsFinished("", False)
|
||||
except Py4JJavaError:
|
||||
# raise it to outside try except
|
||||
raise
|
||||
except:
|
||||
exception = traceback.format_exc()
|
||||
m = re.search("File \"<stdin>\", line (\d+).*", exception)
|
||||
if m:
|
||||
line_no = int(m.group(1))
|
||||
intp.setStatementsFinished(
|
||||
"Fail to execute line {}: {}\n".format(line_no, stmts[line_no - 1]) + exception, True)
|
||||
else:
|
||||
intp.setStatementsFinished(exception, True)
|
||||
else:
|
||||
intp.setStatementsFinished("", False)
|
||||
|
||||
except Py4JJavaError:
|
||||
excInnerError = traceback.format_exc() # format_tb() does not return the inner exception
|
||||
innerErrorStart = excInnerError.find("Py4JJavaError:")
|
||||
|
|
|
|||
47
spark/src/test/resources/log4j.properties
Normal file
47
spark/src/test/resources/log4j.properties
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# Direct log messages to stdout
|
||||
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||
log4j.appender.stdout.Target=System.out
|
||||
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||
log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
|
||||
#log4j.appender.stdout.layout.ConversionPattern=
|
||||
#%5p [%t] (%F:%L) - %m%n
|
||||
#%-4r [%t] %-5p %c %x - %m%n
|
||||
#
|
||||
|
||||
# Root logger option
|
||||
log4j.rootLogger=INFO, stdout
|
||||
|
||||
#mute some noisy guys
|
||||
log4j.logger.org.apache.hadoop.mapred=WARN
|
||||
log4j.logger.org.apache.hadoop.hive.ql=WARN
|
||||
log4j.logger.org.apache.hadoop.hive.metastore=WARN
|
||||
log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
|
||||
log4j.logger.org.apache.zeppelin.scheduler=WARN
|
||||
|
||||
log4j.logger.org.quartz=WARN
|
||||
log4j.logger.DataNucleus=WARN
|
||||
log4j.logger.DataNucleus.MetaData=ERROR
|
||||
log4j.logger.DataNucleus.Datastore=ERROR
|
||||
|
||||
# Log all JDBC parameters
|
||||
log4j.logger.org.hibernate.type=ALL
|
||||
|
||||
log4j.logger.org.apache.zeppelin.interpreter=DEBUG
|
||||
log4j.logger.org.apache.zeppelin.spark=DEBUG
|
||||
|
|
@ -269,7 +269,8 @@ The text of each license is also included at licenses/LICENSE-[project]-[version
|
|||
(The MIT License) Java LSH 0.10 (info.debatty:java-lsh:0.10 - https://github.com/tdebatty/java-LSH)
|
||||
(The MIT License) JSoup 1.6.1 (org.jsoup:jsoup:1.6.1 - https://github.com/jhy/jsoup/)
|
||||
(The MIT License) Unirest 1.4.9 (com.mashape.unirest:unirest-java:1.4.9 - https://github.com/Mashape/unirest-java)
|
||||
(The MIT License) ngclipboard v1.1.1 (https://github.com/sachinchoolur/ngclipboard) - https://github.com/sachinchoolur/ngclipboard/blob/1.1.1/LICENSE)
|
||||
(The MIT License) ngclipboard v1.1.1 (https://github.com/sachinchoolur/ngclipboard) - https://github.com/sachinchoolur/ngclipboard/blob/1.1.1/LICENSE
|
||||
(The MIT License) headroom.js 0.9.3 (https://github.com/WickyNilliams/headroom.js) - https://github.com/WickyNilliams/headroom.js/blob/master/LICENSE
|
||||
|
||||
========================================================================
|
||||
BSD-style licenses
|
||||
|
|
|
|||
|
|
@ -15,104 +15,60 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.zeppelin.spark;
|
||||
package org.apache.zeppelin.interpreter;
|
||||
|
||||
import static scala.collection.JavaConversions.asJavaCollection;
|
||||
import static scala.collection.JavaConversions.asJavaIterable;
|
||||
import static scala.collection.JavaConversions.collectionAsScalaIterable;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.apache.spark.SparkContext;
|
||||
import org.apache.spark.sql.SQLContext;
|
||||
import org.apache.spark.sql.catalyst.expressions.Attribute;
|
||||
import org.apache.zeppelin.annotation.ZeppelinApi;
|
||||
import org.apache.zeppelin.annotation.Experimental;
|
||||
import org.apache.zeppelin.annotation.ZeppelinApi;
|
||||
import org.apache.zeppelin.display.AngularObject;
|
||||
import org.apache.zeppelin.display.AngularObjectRegistry;
|
||||
import org.apache.zeppelin.display.AngularObjectWatcher;
|
||||
import org.apache.zeppelin.display.GUI;
|
||||
import org.apache.zeppelin.display.ui.OptionInput.ParamOption;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContext;
|
||||
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
|
||||
import org.apache.zeppelin.interpreter.InterpreterException;
|
||||
import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
|
||||
import org.apache.zeppelin.interpreter.RemoteWorksController;
|
||||
import org.apache.zeppelin.interpreter.ResultMessages;
|
||||
import org.apache.zeppelin.interpreter.remote.RemoteEventClientWrapper;
|
||||
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
|
||||
import org.apache.zeppelin.resource.Resource;
|
||||
import org.apache.zeppelin.resource.ResourcePool;
|
||||
import org.apache.zeppelin.resource.ResourceSet;
|
||||
|
||||
import scala.Tuple2;
|
||||
import scala.Unit;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Spark context for zeppelin.
|
||||
* Base class for ZeppelinContext
|
||||
*/
|
||||
public class ZeppelinContext {
|
||||
// Map interpreter class name (to be used by hook registry) from
|
||||
// given replName in parapgraph
|
||||
private static final Map<String, String> interpreterClassMap;
|
||||
public abstract class BaseZeppelinContext {
|
||||
|
||||
|
||||
protected InterpreterContext interpreterContext;
|
||||
protected int maxResult;
|
||||
protected InterpreterHookRegistry hooks;
|
||||
protected GUI gui;
|
||||
|
||||
private static RemoteEventClientWrapper eventClient;
|
||||
static {
|
||||
interpreterClassMap = new HashMap<>();
|
||||
interpreterClassMap.put("spark", "org.apache.zeppelin.spark.SparkInterpreter");
|
||||
interpreterClassMap.put("sql", "org.apache.zeppelin.spark.SparkSqlInterpreter");
|
||||
interpreterClassMap.put("dep", "org.apache.zeppelin.spark.DepInterpreter");
|
||||
interpreterClassMap.put("pyspark", "org.apache.zeppelin.spark.PySparkInterpreter");
|
||||
}
|
||||
|
||||
private SparkDependencyResolver dep;
|
||||
private InterpreterContext interpreterContext;
|
||||
private int maxResult;
|
||||
private List<Class> supportedClasses;
|
||||
private InterpreterHookRegistry hooks;
|
||||
|
||||
public ZeppelinContext(SparkContext sc, SQLContext sql,
|
||||
InterpreterContext interpreterContext,
|
||||
SparkDependencyResolver dep,
|
||||
InterpreterHookRegistry hooks,
|
||||
int maxResult) {
|
||||
this.sc = sc;
|
||||
this.sqlContext = sql;
|
||||
this.interpreterContext = interpreterContext;
|
||||
this.dep = dep;
|
||||
|
||||
public BaseZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
|
||||
this.hooks = hooks;
|
||||
this.maxResult = maxResult;
|
||||
this.supportedClasses = new ArrayList<>();
|
||||
try {
|
||||
supportedClasses.add(this.getClass().forName("org.apache.spark.sql.Dataset"));
|
||||
} catch (ClassNotFoundException e) {
|
||||
}
|
||||
|
||||
try {
|
||||
supportedClasses.add(this.getClass().forName("org.apache.spark.sql.DataFrame"));
|
||||
} catch (ClassNotFoundException e) {
|
||||
}
|
||||
|
||||
try {
|
||||
supportedClasses.add(this.getClass().forName("org.apache.spark.sql.SchemaRDD"));
|
||||
} catch (ClassNotFoundException e) {
|
||||
}
|
||||
|
||||
if (supportedClasses.isEmpty()) {
|
||||
throw new InterpreterException("Can not road Dataset/DataFrame/SchemaRDD class");
|
||||
}
|
||||
}
|
||||
|
||||
public SparkContext sc;
|
||||
public SQLContext sqlContext;
|
||||
private GUI gui;
|
||||
// Map interpreter class name (to be used by hook registry) from
|
||||
// given replName in parapgraph
|
||||
public abstract Map<String, String> getInterpreterClassMap();
|
||||
|
||||
public abstract List<Class> getSupportedClasses();
|
||||
|
||||
public int getMaxResult() {
|
||||
return this.maxResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* subclasses should implement this method to display specific data type
|
||||
* @param obj
|
||||
* @return
|
||||
*/
|
||||
protected abstract String showData(Object obj);
|
||||
|
||||
/**
|
||||
* @deprecated use z.textbox instead
|
||||
|
|
@ -143,49 +99,24 @@ public class ZeppelinContext {
|
|||
return gui.textbox(name, defaultValue);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object select(String name, scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
return select(name, "", options);
|
||||
public Object select(String name, Object defaultValue, ParamOption[] paramOptions) {
|
||||
return gui.select(name, defaultValue, paramOptions);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public Object select(String name, Object defaultValue,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
return gui.select(name, defaultValue, tuplesToParamOptions(options));
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public scala.collection.Seq<Object> checkbox(String name,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
List<Object> allChecked = new LinkedList<>();
|
||||
for (Tuple2<Object, String> option : asJavaIterable(options)) {
|
||||
allChecked.add(option._1());
|
||||
public Collection<Object> checkbox(String name, ParamOption[] options) {
|
||||
List<Object> defaultValues = new LinkedList<>();
|
||||
for (ParamOption option : options) {
|
||||
defaultValues.add(option.getValue());
|
||||
}
|
||||
return checkbox(name, collectionAsScalaIterable(allChecked), options);
|
||||
return checkbox(name, defaultValues, options);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public scala.collection.Seq<Object> checkbox(String name,
|
||||
scala.collection.Iterable<Object> defaultChecked,
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
return scala.collection.JavaConversions.asScalaBuffer(
|
||||
gui.checkbox(name, asJavaCollection(defaultChecked),
|
||||
tuplesToParamOptions(options))).toSeq();
|
||||
}
|
||||
|
||||
private ParamOption[] tuplesToParamOptions(
|
||||
scala.collection.Iterable<Tuple2<Object, String>> options) {
|
||||
int n = options.size();
|
||||
ParamOption[] paramOptions = new ParamOption[n];
|
||||
Iterator<Tuple2<Object, String>> it = asJavaIterable(options).iterator();
|
||||
|
||||
int i = 0;
|
||||
while (it.hasNext()) {
|
||||
Tuple2<Object, String> valueAndDisplayValue = it.next();
|
||||
paramOptions[i++] = new ParamOption(valueAndDisplayValue._1(), valueAndDisplayValue._2());
|
||||
}
|
||||
|
||||
return paramOptions;
|
||||
public Collection<Object> checkbox(String name,
|
||||
List<Object> defaultValues,
|
||||
ParamOption[] options) {
|
||||
return gui.checkbox(name, defaultValues, options);
|
||||
}
|
||||
|
||||
public void setGui(GUI o) {
|
||||
|
|
@ -208,8 +139,9 @@ public class ZeppelinContext {
|
|||
}
|
||||
|
||||
/**
|
||||
* show DataFrame or SchemaRDD
|
||||
* @param o DataFrame or SchemaRDD object
|
||||
* display special types of objects for interpreter.
|
||||
* Each interpreter can has its own supported classes.
|
||||
* @param o object
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public void show(Object o) {
|
||||
|
|
@ -217,16 +149,17 @@ public class ZeppelinContext {
|
|||
}
|
||||
|
||||
/**
|
||||
* show DataFrame or SchemaRDD
|
||||
* @param o DataFrame or SchemaRDD object
|
||||
* display special types of objects for interpreter.
|
||||
* Each interpreter can has its own supported classes.
|
||||
* @param o object
|
||||
* @param maxResult maximum number of rows to display
|
||||
*/
|
||||
|
||||
@ZeppelinApi
|
||||
public void show(Object o, int maxResult) {
|
||||
try {
|
||||
if (supportedClasses.contains(o.getClass())) {
|
||||
interpreterContext.out.write(showDF(sc, interpreterContext, o, maxResult));
|
||||
if (isSupportedObject(o)) {
|
||||
interpreterContext.out.write(showData(o));
|
||||
} else {
|
||||
interpreterContext.out.write(o.toString());
|
||||
}
|
||||
|
|
@ -235,93 +168,13 @@ public class ZeppelinContext {
|
|||
}
|
||||
}
|
||||
|
||||
public static String showDF(ZeppelinContext z, Object df) {
|
||||
return showDF(z.sc, z.interpreterContext, df, z.maxResult);
|
||||
}
|
||||
|
||||
public static String showDF(SparkContext sc,
|
||||
InterpreterContext interpreterContext,
|
||||
Object df, int maxResult) {
|
||||
Object[] rows = null;
|
||||
Method take;
|
||||
String jobGroup = Utils.buildJobGroupId(interpreterContext);
|
||||
sc.setJobGroup(jobGroup, "Zeppelin", false);
|
||||
|
||||
try {
|
||||
// convert it to DataFrame if it is Dataset, as we will iterate all the records
|
||||
// and assume it is type Row.
|
||||
if (df.getClass().getCanonicalName().equals("org.apache.spark.sql.Dataset")) {
|
||||
Method convertToDFMethod = df.getClass().getMethod("toDF");
|
||||
df = convertToDFMethod.invoke(df);
|
||||
private boolean isSupportedObject(Object obj) {
|
||||
for (Class supportedClass : getSupportedClasses()) {
|
||||
if (supportedClass.isInstance(obj)) {
|
||||
return true;
|
||||
}
|
||||
take = df.getClass().getMethod("take", int.class);
|
||||
rows = (Object[]) take.invoke(df, maxResult + 1);
|
||||
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
|
||||
| IllegalArgumentException | InvocationTargetException | ClassCastException e) {
|
||||
sc.clearJobGroup();
|
||||
throw new InterpreterException(e);
|
||||
}
|
||||
|
||||
List<Attribute> columns = null;
|
||||
// get field names
|
||||
try {
|
||||
// Use reflection because of classname returned by queryExecution changes from
|
||||
// Spark <1.5.2 org.apache.spark.sql.SQLContext$QueryExecution
|
||||
// Spark 1.6.0> org.apache.spark.sql.hive.HiveContext$QueryExecution
|
||||
Object qe = df.getClass().getMethod("queryExecution").invoke(df);
|
||||
Object a = qe.getClass().getMethod("analyzed").invoke(qe);
|
||||
scala.collection.Seq seq = (scala.collection.Seq) a.getClass().getMethod("output").invoke(a);
|
||||
|
||||
columns = (List<Attribute>) scala.collection.JavaConverters.seqAsJavaListConverter(seq)
|
||||
.asJava();
|
||||
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
|
||||
| IllegalArgumentException | InvocationTargetException e) {
|
||||
throw new InterpreterException(e);
|
||||
}
|
||||
|
||||
StringBuilder msg = new StringBuilder();
|
||||
msg.append("%table ");
|
||||
for (Attribute col : columns) {
|
||||
msg.append(col.name() + "\t");
|
||||
}
|
||||
String trim = msg.toString().trim();
|
||||
msg = new StringBuilder(trim);
|
||||
msg.append("\n");
|
||||
|
||||
// ArrayType, BinaryType, BooleanType, ByteType, DecimalType, DoubleType, DynamicType,
|
||||
// FloatType, FractionalType, IntegerType, IntegralType, LongType, MapType, NativeType,
|
||||
// NullType, NumericType, ShortType, StringType, StructType
|
||||
|
||||
try {
|
||||
for (int r = 0; r < maxResult && r < rows.length; r++) {
|
||||
Object row = rows[r];
|
||||
Method isNullAt = row.getClass().getMethod("isNullAt", int.class);
|
||||
Method apply = row.getClass().getMethod("apply", int.class);
|
||||
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
if (!(Boolean) isNullAt.invoke(row, i)) {
|
||||
msg.append(apply.invoke(row, i).toString());
|
||||
} else {
|
||||
msg.append("null");
|
||||
}
|
||||
if (i != columns.size() - 1) {
|
||||
msg.append("\t");
|
||||
}
|
||||
}
|
||||
msg.append("\n");
|
||||
}
|
||||
} catch (NoSuchMethodException | SecurityException | IllegalAccessException
|
||||
| IllegalArgumentException | InvocationTargetException e) {
|
||||
throw new InterpreterException(e);
|
||||
}
|
||||
|
||||
if (rows.length > maxResult) {
|
||||
msg.append("\n");
|
||||
msg.append(ResultMessages.getExceedsLimitRowsMessage(maxResult,
|
||||
SparkSqlInterpreter.MAX_RESULTS));
|
||||
}
|
||||
sc.clearJobGroup();
|
||||
return msg.toString();
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -664,31 +517,7 @@ public class ZeppelinContext {
|
|||
angularWatch(name, null, watcher);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public void angularWatch(String name,
|
||||
final scala.Function2<Object, Object, Unit> func) {
|
||||
angularWatch(name, interpreterContext.getNoteId(), func);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void angularWatchGlobal(String name,
|
||||
final scala.Function2<Object, Object, Unit> func) {
|
||||
angularWatch(name, null, func);
|
||||
}
|
||||
|
||||
@ZeppelinApi
|
||||
public void angularWatch(
|
||||
String name,
|
||||
final scala.Function3<Object, Object, InterpreterContext, Unit> func) {
|
||||
angularWatch(name, interpreterContext.getNoteId(), func);
|
||||
}
|
||||
|
||||
@Deprecated
|
||||
public void angularWatchGlobal(
|
||||
String name,
|
||||
final scala.Function3<Object, Object, InterpreterContext, Unit> func) {
|
||||
angularWatch(name, null, func);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove watcher from angular variable (local)
|
||||
|
|
@ -788,7 +617,7 @@ public class ZeppelinContext {
|
|||
* @param name name of the variable
|
||||
* @param watcher watcher
|
||||
*/
|
||||
private void angularWatch(String name, String noteId, AngularObjectWatcher watcher) {
|
||||
public void angularWatch(String name, String noteId, AngularObjectWatcher watcher) {
|
||||
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();
|
||||
|
||||
if (registry.get(name, noteId, null) != null) {
|
||||
|
|
@ -796,33 +625,6 @@ public class ZeppelinContext {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
private void angularWatch(String name, String noteId,
|
||||
final scala.Function2<Object, Object, Unit> func) {
|
||||
AngularObjectWatcher w = new AngularObjectWatcher(getInterpreterContext()) {
|
||||
@Override
|
||||
public void watch(Object oldObject, Object newObject,
|
||||
InterpreterContext context) {
|
||||
func.apply(newObject, newObject);
|
||||
}
|
||||
};
|
||||
angularWatch(name, noteId, w);
|
||||
}
|
||||
|
||||
private void angularWatch(
|
||||
String name,
|
||||
String noteId,
|
||||
final scala.Function3<Object, Object, InterpreterContext, Unit> func) {
|
||||
AngularObjectWatcher w = new AngularObjectWatcher(getInterpreterContext()) {
|
||||
@Override
|
||||
public void watch(Object oldObject, Object newObject,
|
||||
InterpreterContext context) {
|
||||
func.apply(oldObject, newObject, context);
|
||||
}
|
||||
};
|
||||
angularWatch(name, noteId, w);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove watcher
|
||||
* @param name
|
||||
|
|
@ -860,16 +662,16 @@ public class ZeppelinContext {
|
|||
* @param replName if replName is a valid className, return that instead.
|
||||
*/
|
||||
public String getClassNameFromReplName(String replName) {
|
||||
for (String name : interpreterClassMap.values()) {
|
||||
for (String name : getInterpreterClassMap().keySet()) {
|
||||
if (replName.equals(name)) {
|
||||
return replName;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (replName.contains("spark.")) {
|
||||
replName = replName.replace("spark.", "");
|
||||
}
|
||||
return interpreterClassMap.get(replName);
|
||||
return getInterpreterClassMap().get(replName);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1012,8 +814,8 @@ public class ZeppelinContext {
|
|||
*/
|
||||
@ZeppelinApi
|
||||
public void setEventClient(RemoteEventClientWrapper eventClient) {
|
||||
if (ZeppelinContext.eventClient == null) {
|
||||
ZeppelinContext.eventClient = eventClient;
|
||||
if (BaseZeppelinContext.eventClient == null) {
|
||||
BaseZeppelinContext.eventClient = eventClient;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -64,6 +64,19 @@ public abstract class Interpreter {
|
|||
@ZeppelinApi
|
||||
public abstract void close();
|
||||
|
||||
/**
|
||||
* Run precode if exists.
|
||||
*/
|
||||
@ZeppelinApi
|
||||
public InterpreterResult executePrecode(InterpreterContext interpreterContext) {
|
||||
String simpleName = this.getClass().getSimpleName();
|
||||
String precode = getProperty(String.format("zeppelin.%s.precode", simpleName));
|
||||
if (StringUtils.isNotBlank(precode)) {
|
||||
return interpret(precode, interpreterContext);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run code and return result, in synchronous way.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ import java.net.URL;
|
|||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.apache.zeppelin.interpreter.remote.RemoteInterpreter;
|
||||
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
|
||||
import org.apache.zeppelin.scheduler.Scheduler;
|
||||
|
||||
|
|
@ -73,6 +72,11 @@ public class LazyOpenInterpreter
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InterpreterResult executePrecode(InterpreterContext interpreterContext) {
|
||||
return intp.executePrecode(interpreterContext);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
synchronized (intp) {
|
||||
|
|
@ -157,7 +161,7 @@ public class LazyOpenInterpreter
|
|||
public void setClassloaderUrls(URL [] urls) {
|
||||
intp.setClassloaderUrls(urls);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void registerHook(String noteId, String event, String cmd) {
|
||||
intp.registerHook(noteId, event, cmd);
|
||||
|
|
|
|||
|
|
@ -481,19 +481,24 @@ public class RemoteInterpreterServer
|
|||
try {
|
||||
InterpreterContext.set(context);
|
||||
|
||||
InterpreterResult result = null;
|
||||
|
||||
// Open the interpreter instance prior to calling interpret().
|
||||
// This is necessary because the earliest we can register a hook
|
||||
// is from within the open() method.
|
||||
LazyOpenInterpreter lazy = (LazyOpenInterpreter) interpreter;
|
||||
if (!lazy.isOpen()) {
|
||||
lazy.open();
|
||||
result = lazy.executePrecode(context);
|
||||
}
|
||||
|
||||
// Add hooks to script from registry.
|
||||
// Global scope first, followed by notebook scope
|
||||
processInterpreterHooks(null);
|
||||
processInterpreterHooks(context.getNoteId());
|
||||
InterpreterResult result = interpreter.interpret(script, context);
|
||||
if (result == null || result.code() == Code.SUCCESS) {
|
||||
// Add hooks to script from registry.
|
||||
// Global scope first, followed by notebook scope
|
||||
processInterpreterHooks(null);
|
||||
processInterpreterHooks(context.getNoteId());
|
||||
result = interpreter.interpret(script, context);
|
||||
}
|
||||
|
||||
// data from context.out is prepended to InterpreterResult if both defined
|
||||
context.out.flush();
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import java.io.IOException;
|
|||
* Can be used to channel output from interpreters.
|
||||
*/
|
||||
public class InterpreterOutputStream extends LogOutputStream {
|
||||
public static Logger logger;
|
||||
private Logger logger;
|
||||
InterpreterOutput interpreterOutput;
|
||||
boolean ignoreLeadingNewLinesFromScalaReporter = false;
|
||||
|
||||
|
|
|
|||
|
|
@ -170,6 +170,79 @@ public class RemoteInterpreterTest {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteIncorrectPrecode() throws TTransportException, IOException {
|
||||
Properties p = new Properties();
|
||||
p.put("zeppelin.MockInterpreterA.precode", "fail test");
|
||||
intpGroup.put("note", new LinkedList<Interpreter>());
|
||||
|
||||
RemoteInterpreter intpA = createMockInterpreterA(p);
|
||||
|
||||
intpGroup.get("note").add(intpA);
|
||||
|
||||
intpA.setInterpreterGroup(intpGroup);
|
||||
|
||||
RemoteInterpreterProcess process = intpA.getInterpreterProcess();
|
||||
|
||||
intpA.open();
|
||||
|
||||
InterpreterResult result = intpA.interpret("1",
|
||||
new InterpreterContext(
|
||||
"note",
|
||||
"id",
|
||||
null,
|
||||
"title",
|
||||
"text",
|
||||
new AuthenticationInfo(),
|
||||
new HashMap<String, Object>(),
|
||||
new GUI(),
|
||||
new AngularObjectRegistry(intpGroup.getId(), null),
|
||||
new LocalResourcePool("pool1"),
|
||||
new LinkedList<InterpreterContextRunner>(), null));
|
||||
|
||||
|
||||
|
||||
intpA.close();
|
||||
assertEquals(Code.ERROR, result.code());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExecuteCorrectPrecode() throws TTransportException, IOException {
|
||||
Properties p = new Properties();
|
||||
p.put("zeppelin.MockInterpreterA.precode", "2");
|
||||
intpGroup.put("note", new LinkedList<Interpreter>());
|
||||
|
||||
RemoteInterpreter intpA = createMockInterpreterA(p);
|
||||
|
||||
intpGroup.get("note").add(intpA);
|
||||
|
||||
intpA.setInterpreterGroup(intpGroup);
|
||||
|
||||
RemoteInterpreterProcess process = intpA.getInterpreterProcess();
|
||||
|
||||
intpA.open();
|
||||
|
||||
InterpreterResult result = intpA.interpret("1",
|
||||
new InterpreterContext(
|
||||
"note",
|
||||
"id",
|
||||
null,
|
||||
"title",
|
||||
"text",
|
||||
new AuthenticationInfo(),
|
||||
new HashMap<String, Object>(),
|
||||
new GUI(),
|
||||
new AngularObjectRegistry(intpGroup.getId(), null),
|
||||
new LocalResourcePool("pool1"),
|
||||
new LinkedList<InterpreterContextRunner>(), null));
|
||||
|
||||
|
||||
|
||||
intpA.close();
|
||||
assertEquals(Code.SUCCESS, result.code());
|
||||
assertEquals("1", result.message().get(0).getData());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoteInterperterErrorStatus() throws TTransportException, IOException {
|
||||
Properties p = new Properties();
|
||||
|
|
|
|||
|
|
@ -275,6 +275,27 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
|
|||
waitForFinish(p);
|
||||
assertEquals(Status.FINISHED, p.getStatus());
|
||||
assertEquals("[Row(len=u'3')]\n", p.getResult().message().get(0).getData());
|
||||
|
||||
// test exception
|
||||
p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
|
||||
config = p.getConfig();
|
||||
config.put("enabled", true);
|
||||
p.setConfig(config);
|
||||
/**
|
||||
%pyspark
|
||||
a=1
|
||||
|
||||
print(a2)
|
||||
*/
|
||||
p.setText("%pyspark a=1\n\nprint(a2)");
|
||||
p.setAuthenticationInfo(anonymous);
|
||||
note.run(p.getId());
|
||||
waitForFinish(p);
|
||||
assertEquals(Status.ERROR, p.getStatus());
|
||||
assertTrue(p.getResult().message().get(0).getData()
|
||||
.contains("Fail to execute line 3: print(a2)"));
|
||||
assertTrue(p.getResult().message().get(0).getData()
|
||||
.contains("name 'a2' is not defined"));
|
||||
}
|
||||
if (sparkVersion >= 20) {
|
||||
// run SparkSession test
|
||||
|
|
|
|||
|
|
@ -43,4 +43,5 @@ log4j.logger.DataNucleus.Datastore=ERROR
|
|||
# Log all JDBC parameters
|
||||
log4j.logger.org.hibernate.type=ALL
|
||||
|
||||
log4j.logger.org.apache.zeppelin.interpreter=DEBUG
|
||||
log4j.logger.org.apache.zeppelin.interpreter=DEBUG
|
||||
log4j.logger.org.apache.zeppelin.spark=DEBUG
|
||||
|
|
|
|||
|
|
@ -15,14 +15,6 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Generated on 2014-08-29 using generator-angular 0.9.5
|
||||
|
||||
// # Globbing
|
||||
// for performance reasons we're only matching one level down:
|
||||
// 'test/spec/{,*/}*.js'
|
||||
// use this if you want to recursively match all subfolders:
|
||||
// 'test/spec/**/*.js'
|
||||
|
||||
module.exports = function(grunt) {
|
||||
|
||||
// Load grunt tasks automatically
|
||||
|
|
@ -47,22 +39,6 @@ module.exports = function(grunt) {
|
|||
// Project settings
|
||||
yeoman: appConfig,
|
||||
|
||||
babel: {
|
||||
options: {
|
||||
sourceMap: true,
|
||||
presets: ['es2015'],
|
||||
plugins: ['transform-object-rest-spread']
|
||||
},
|
||||
dist: {
|
||||
files: [{
|
||||
expand: true,
|
||||
cwd: '.tmp/concat/scripts',
|
||||
src: ['scripts.js'],
|
||||
dest: '.tmp/concat/scripts',
|
||||
}]
|
||||
}
|
||||
},
|
||||
|
||||
// use ngAnnotate instead og ngMin
|
||||
ngAnnotate: {
|
||||
dist: {
|
||||
|
|
@ -102,7 +78,7 @@ module.exports = function(grunt) {
|
|||
watch: {
|
||||
bower: {
|
||||
files: ['bower.json'],
|
||||
tasks: ['wiredep']
|
||||
tasks: ['wiredep:dist', 'wiredep:test']
|
||||
},
|
||||
html: {
|
||||
files: [
|
||||
|
|
@ -160,15 +136,24 @@ module.exports = function(grunt) {
|
|||
|
||||
// Automatically inject Bower components into the app
|
||||
wiredep: {
|
||||
options: {},
|
||||
app: {
|
||||
ci: {
|
||||
src: ['<%= yeoman.app %>/index.html'],
|
||||
ignorePath: /\.\.\//
|
||||
ignorePath: /\.\.\//,
|
||||
exclude: [
|
||||
]
|
||||
},
|
||||
dist: {
|
||||
src: ['<%= yeoman.app %>/index.html'],
|
||||
ignorePath: /\.\.\//,
|
||||
exclude: [
|
||||
],
|
||||
},
|
||||
test: {
|
||||
devDependencies: true,
|
||||
src: '<%= karma.unit.configFile %>',
|
||||
ignorePath: /\.\.\//,
|
||||
exclude: [
|
||||
],
|
||||
fileTypes: {
|
||||
js: {
|
||||
block: /(([\s\t]*)\/{2}\s*?bower:\s*?(\S*))(\n|\r|.)*?(\/{2}\s*endbower)/gi,
|
||||
|
|
@ -387,7 +372,7 @@ module.exports = function(grunt) {
|
|||
// Test settings
|
||||
karma: {
|
||||
unit: {
|
||||
configFile: 'test/karma.conf.js',
|
||||
configFile: 'karma.conf.js',
|
||||
singleRun: true
|
||||
}
|
||||
}
|
||||
|
|
@ -395,7 +380,8 @@ module.exports = function(grunt) {
|
|||
|
||||
grunt.registerTask('pre-webpack-dev', 'Compile then start a connect web server', function(target) {
|
||||
grunt.task.run([
|
||||
'wiredep',
|
||||
'wiredep:test',
|
||||
'wiredep:dist',
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
@ -405,7 +391,14 @@ module.exports = function(grunt) {
|
|||
|
||||
grunt.registerTask('pre-webpack-dist', [
|
||||
'htmlhint',
|
||||
'wiredep',
|
||||
'wiredep:test',
|
||||
'wiredep:dist',
|
||||
]);
|
||||
|
||||
grunt.registerTask('pre-webpack-ci', [
|
||||
'htmlhint',
|
||||
'wiredep:test',
|
||||
'wiredep:ci',
|
||||
]);
|
||||
|
||||
grunt.registerTask('post-webpack-dist', [
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@
|
|||
"clean": "rimraf dist && rimraf .tmp",
|
||||
"postinstall": "bower install --silent",
|
||||
"prebuild": "npm-run-all clean lint:once",
|
||||
"build": "grunt pre-webpack-dist && webpack && grunt post-webpack-dist",
|
||||
"build:dist": "grunt pre-webpack-dist && webpack && grunt post-webpack-dist",
|
||||
"build:ci": "grunt pre-webpack-ci && webpack && grunt post-webpack-dist",
|
||||
"lint:watch": "esw --watch src",
|
||||
"lint:once": "eslint src",
|
||||
"predev": "grunt pre-webpack-dev",
|
||||
|
|
@ -22,7 +23,8 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"grunt-angular-templates": "^0.5.7",
|
||||
"grunt-dom-munger": "^3.4.0"
|
||||
"grunt-dom-munger": "^3.4.0",
|
||||
"headroom.js": "^0.9.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"autoprefixer": "^6.5.4",
|
||||
|
|
@ -64,6 +66,7 @@
|
|||
"grunt-usemin": "^2.1.1",
|
||||
"grunt-wiredep": "~2.0.0",
|
||||
"html-webpack-plugin": "^2.24.1",
|
||||
"imports-loader": "^0.7.1",
|
||||
"jasmine-core": "^2.5.2",
|
||||
"karma": "~1.3.0",
|
||||
"karma-coverage": "^1.1.1",
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@
|
|||
<goal>yarn</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<arguments>run build</arguments>
|
||||
<arguments>${web.build.command}</arguments>
|
||||
</configuration>
|
||||
</execution>
|
||||
|
||||
|
|
@ -154,5 +154,23 @@
|
|||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>web-dist</id>
|
||||
<activation>
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<web.build.command>run build:dist</web.build.command>
|
||||
</properties>
|
||||
</profile>
|
||||
<profile>
|
||||
<id>web-ci</id>
|
||||
<properties>
|
||||
<web.build.command>run build:ci</web.build.command>
|
||||
</properties>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -15,7 +15,10 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
let zeppelinWebApp = angular.module('zeppelinWebApp', [
|
||||
import 'headroom.js'
|
||||
import 'headroom.js/dist/angular.headroom'
|
||||
|
||||
const requiredModules = [
|
||||
'ngCookies',
|
||||
'ngAnimate',
|
||||
'ngRoute',
|
||||
|
|
@ -33,8 +36,14 @@ let zeppelinWebApp = angular.module('zeppelinWebApp', [
|
|||
'ngToast',
|
||||
'focus-if',
|
||||
'ngResource',
|
||||
'ngclipboard'
|
||||
])
|
||||
'ngclipboard',
|
||||
]
|
||||
|
||||
// headroom should not be used for CI, since we have to execute some integration tests.
|
||||
// otherwise, they will fail.
|
||||
if (!process.env.BUILD_CI) { requiredModules.push('headroom') }
|
||||
|
||||
let zeppelinWebApp = angular.module('zeppelinWebApp', requiredModules)
|
||||
.filter('breakFilter', function () {
|
||||
return function (text) {
|
||||
// eslint-disable-next-line no-extra-boolean-cast
|
||||
|
|
|
|||
|
|
@ -133,6 +133,7 @@ a.navbar-brand:hover {
|
|||
.dropdown-menu > .scrollbar-container > li > a,
|
||||
.dropdown-menu .notebook-list-item {
|
||||
display: block;
|
||||
text-decoration: none;
|
||||
padding: 1px 10px;
|
||||
clear: both;
|
||||
font-weight: normal;
|
||||
|
|
@ -294,7 +295,7 @@ a.navbar-brand:hover {
|
|||
#notebook-list {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
display: inline;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
@media (min-width: 768px) {
|
||||
|
|
|
|||
|
|
@ -58,14 +58,12 @@ limitations under the License.
|
|||
<a style="text-decoration: none; cursor: pointer;" ng-click="toggleFolderNode(node)">
|
||||
<i style="font-size: 10px;" ng-class="node.hidden ? 'icon-folder' : 'icon-folder-alt'"></i> {{getNoteName(node)}}
|
||||
</a>
|
||||
<a ng-if="!node.isTrash" style="text-decoration: none;">
|
||||
<a href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;"
|
||||
ng-controller="NotenameCtrl as notenamectrl" ng-click="notenamectrl.getInterpreterSettings()" data-path="{{node.id}}">
|
||||
<i style="margin-left: 10px;"
|
||||
class="fa fa-plus notebook-list-btn" ng-show="showFolderButton"
|
||||
tooltip-placement="bottom" uib-tooltip="Create new note">
|
||||
</i>
|
||||
</a>
|
||||
<a ng-if="!node.isTrash" href="" data-toggle="modal" data-target="#noteNameModal" style="text-decoration: none;"
|
||||
ng-controller="NotenameCtrl as notenamectrl" ng-click="notenamectrl.getInterpreterSettings()" data-path="{{node.id}}">
|
||||
<i style="margin-left: 10px;"
|
||||
class="fa fa-plus notebook-list-btn" ng-show="showFolderButton"
|
||||
tooltip-placement="bottom" uib-tooltip="Create new note">
|
||||
</i>
|
||||
</a>
|
||||
<a ng-if="!node.isTrash" style="text-decoration: none;">
|
||||
<i class="fa fa-pencil notebook-list-btn" ng-show="showFolderButton" ng-click="renameFolder(node)"
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<div class="noteAction" ng-show="note.id && !paragraphUrl">
|
||||
<headroom class="noteAction"
|
||||
ng-show="note.id && !paragraphUrl">
|
||||
<h3>
|
||||
<div style="float: left; width: auto; max-width: 40%"
|
||||
ng-controller="ElasticInputCtrl as input">
|
||||
|
|
@ -274,4 +275,4 @@ limitations under the License.
|
|||
</span>
|
||||
</div>
|
||||
</h3>
|
||||
</div>
|
||||
</headroom>
|
||||
|
|
|
|||
|
|
@ -320,3 +320,14 @@
|
|||
pointer-events: none;
|
||||
}
|
||||
|
||||
/** required to pin, unpin `noteAction` */
|
||||
.noteAction.headroom {
|
||||
position: fixed;
|
||||
top: 50px;
|
||||
left: 0;
|
||||
right: 0;
|
||||
transition: all .2s ease-in-out;
|
||||
}
|
||||
.noteAction.headroom--unpinned { top: -100px; }
|
||||
.noteAction.headroom--pinned { top: 50px; /** `noteAction` top */ }
|
||||
|
||||
|
|
|
|||
|
|
@ -1331,7 +1331,8 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca
|
|||
$scope.$emit('moveFocusToPreviousParagraph', paragraphId)
|
||||
} else if (editorHide && (keyCode === 40 || (keyCode === 78 && keyEvent.ctrlKey && !keyEvent.altKey))) { // down
|
||||
// move focus to next paragraph
|
||||
$scope.$emit('moveFocusToNextParagraph', paragraphId)
|
||||
// $timeout stops chaining effect of focus propogation
|
||||
$timeout(() => $scope.$emit('moveFocusToNextParagraph', paragraphId))
|
||||
} else if (keyEvent.shiftKey && keyCode === 13) { // Shift + Enter
|
||||
$scope.runParagraphFromShortcut($scope.getEditorValue())
|
||||
} else if (keyEvent.ctrlKey && keyEvent.altKey && keyCode === 67) { // Ctrl + Alt + c
|
||||
|
|
|
|||
|
|
@ -36,4 +36,3 @@ body {
|
|||
.ng-toast.ng-toast--top {
|
||||
top: 100px;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,15 @@ function expandCollapse () {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
let target = event.target
|
||||
|
||||
// add note
|
||||
if (target.classList !== undefined && target.classList.contains('fa-plus') &&
|
||||
target.tagName.toLowerCase() === 'i') {
|
||||
return
|
||||
}
|
||||
|
||||
event.stopPropagation()
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,20 +12,27 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
-->
|
||||
|
||||
<a class="notebook-list-item" ng-if="navbar.isFilterNote(note) && !note.children" href="#/notebook/{{note.id}}">
|
||||
<i style="font-size: 10px; margin-right: 5px;" ng-class="query.q && note.isTrash ? 'fa fa-trash-o' : 'icon-doc'" ></i>
|
||||
<span>{{noteName(note)}}</span>
|
||||
<a class="notebook-list-item" ng-if="navbar.isFilterNote(node) && !node.children" href="#/notebook/{{node.id}}">
|
||||
<i style="font-size: 10px; margin-right: 5px;" ng-class="query.q && node.isTrash ? 'fa fa-trash-o' : 'icon-doc'" ></i>
|
||||
<span>{{noteName(node)}}</span>
|
||||
</a>
|
||||
|
||||
<li ng-if="note.children" ng-click="$event.stopPropagation()">
|
||||
<li ng-if="node.children">
|
||||
<expand-collapse>
|
||||
<div>
|
||||
<div ng-mouseenter="showFolderButton=true" ng-mouseleave="showFolderButton=false">
|
||||
<a class="notebook-list-item" href="javascript:void(0)">
|
||||
<div ng-if="note.id !== navbar.TRASH_FOLDER_ID">
|
||||
<div ng-if="node.id !== navbar.TRASH_FOLDER_ID">
|
||||
<i style="font-size: 10px; margin-right: 5px;" class="icon-folder"></i>
|
||||
<span>{{noteName(note)}}</span>
|
||||
<span>{{noteName(node)}}</span>
|
||||
<i data-toggle="modal" data-target="#noteNameModal" ng-controller="NotenameCtrl as notenamectrl"
|
||||
ng-click="notenamectrl.getInterpreterSettings()" data-path="{{node.id}}"
|
||||
style="font-size: 12px; margin-left: 5px; margin-right: 5px;"
|
||||
ng-show="showFolderButton" class="fa fa-plus"
|
||||
uib-tooltip="Create new note"
|
||||
tooltip-placement="{{calculateTooltipPlacement(node)}}">
|
||||
</i>
|
||||
</div>
|
||||
<div ng-if="note.id === navbar.TRASH_FOLDER_ID">
|
||||
<div ng-if="node.id === navbar.TRASH_FOLDER_ID">
|
||||
<i style="font-size: 12px; margin-right: 5px;" class="fa fa-trash-o"></i>
|
||||
<span>Trash</span>
|
||||
</div>
|
||||
|
|
@ -33,8 +40,8 @@ limitations under the License.
|
|||
</div>
|
||||
<div class="expandable" style="color: black;">
|
||||
<ul>
|
||||
<li ng-repeat="note in note.children | orderBy:node:false:navbar.arrayOrderingSrv.noteComparator track by $index"
|
||||
ng-class="{'active' : navbar.isActive(note.id)}"
|
||||
<li ng-repeat="node in node.children | orderBy:node:false:navbar.arrayOrderingSrv.noteComparator track by $index"
|
||||
ng-class="{'active' : navbar.isActive(node.id)}"
|
||||
ng-include="'components/navbar/navbar-noteList-elem.html'">
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -150,4 +150,16 @@ function NavCtrl ($scope, $rootScope, $http, $routeParams, $location,
|
|||
})
|
||||
})
|
||||
}
|
||||
|
||||
$scope.calculateTooltipPlacement = function (note) {
|
||||
if (note !== undefined && note.name !== undefined) {
|
||||
let length = note.name.length
|
||||
if (length < 2) {
|
||||
return 'top-left'
|
||||
} else if (length > 7) {
|
||||
return 'top-right'
|
||||
}
|
||||
}
|
||||
return 'top'
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,21 +12,32 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#searchTermId {
|
||||
#searchTermId {
|
||||
min-width: 200px;
|
||||
}
|
||||
|
||||
|
||||
@media (min-width: 795px) and (max-width: 830px) {
|
||||
input#searchTermId {
|
||||
input#searchTermId {
|
||||
width: 170px;
|
||||
min-width: 170px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (min-width: 768px) and (max-width: 794px) {
|
||||
input#searchTermId {
|
||||
input#searchTermId {
|
||||
width: 140px;
|
||||
min-width: 140px;
|
||||
}
|
||||
}
|
||||
|
||||
/** required to pin, unpin `navbar-fixed-top` */
|
||||
.navbar-fixed-top.headroom {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
transition: all .2s ease-in-out;
|
||||
}
|
||||
.navbar-fixed-top.headroom--unpinned { top: -100px; }
|
||||
.navbar-fixed-top.headroom--pinned { top: 0; /** `navbar` top */ }
|
||||
|
|
|
|||
|
|
@ -10,7 +10,9 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
-->
|
||||
|
||||
<div class="navbar navbar-inverse navbar-fixed-top" style="display: none;" role="navigation" ng-class="{'displayNavBar': !asIframe}">
|
||||
<headroom class="navbar navbar-inverse navbar-fixed-top"
|
||||
style="display: none;" role="navigation"
|
||||
ng-class="{'displayNavBar': !asIframe}">
|
||||
<div class="container">
|
||||
<div class="navbar-header">
|
||||
<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target=".navbar-collapse">
|
||||
|
|
@ -33,13 +35,13 @@ limitations under the License.
|
|||
<div id="notebook-list" class="scrollbar-container" ng-if="isDrawNavbarNoteList">
|
||||
<li class="filter-names" ng-include="'components/filterNoteNames/filter-note-names.html'"></li>
|
||||
<div ng-if="!query.q || query.q === ''">
|
||||
<li ng-repeat="note in navbar.notes.root.children | orderBy:node:false:navbar.arrayOrderingSrv.noteComparator track by note.id"
|
||||
ng-class="{'active' : navbar.isActive(note.id)}" ng-include="'components/navbar/navbar-noteList-elem.html'">
|
||||
<li ng-repeat="node in navbar.notes.root.children | orderBy:node:false:navbar.arrayOrderingSrv.noteComparator track by node.id"
|
||||
ng-class="{'active' : navbar.isActive(node.id)}" ng-include="'components/navbar/navbar-noteList-elem.html'">
|
||||
</li>
|
||||
</div>
|
||||
<div ng-if="query.q">
|
||||
<li ng-repeat="note in navbar.notes.flatList | filter : query.q | orderBy:navbar.arrayOrderingSrv.noteFlatListOrdering track by note.id"
|
||||
ng-class="{'active' : navbar.isActive(note.id)}" ng-include="'components/navbar/navbar-noteList-elem.html'">
|
||||
<li ng-repeat="node in navbar.notes.flatList | filter : query.q | orderBy:navbar.arrayOrderingSrv.noteFlatListOrdering track by node.id"
|
||||
ng-class="{'active' : navbar.isActive(node.id)}" ng-include="'components/navbar/navbar-noteList-elem.html'">
|
||||
</li>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -109,7 +111,7 @@ limitations under the License.
|
|||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</headroom>
|
||||
<div id="aboutModal" class="modal fade" role="dialog"
|
||||
tabindex="-1">
|
||||
<div class="modal-dialog">
|
||||
|
|
|
|||
|
|
@ -68,7 +68,8 @@ InsertLiveReloadPlugin.prototype.apply = function apply(compiler) {
|
|||
*/
|
||||
var ENV = process.env.npm_lifecycle_event;
|
||||
var isTest = ENV === 'test';
|
||||
var isProd = ENV === 'build';
|
||||
var isProd = ENV.startsWith('build')
|
||||
var isCI = ENV === 'build:ci'
|
||||
|
||||
module.exports = function makeWebpackConfig () {
|
||||
/**
|
||||
|
|
@ -139,6 +140,11 @@ module.exports = function makeWebpackConfig () {
|
|||
config.module = {
|
||||
preLoaders: [],
|
||||
loaders: [{
|
||||
// headroom 0.9.3 doesn't work with webpack
|
||||
// https://github.com/WickyNilliams/headroom.js/issues/213#issuecomment-281106943
|
||||
test: require.resolve('headroom.js'),
|
||||
loader: 'imports-loader?this=>window,define=>false,exports=>false'
|
||||
}, {
|
||||
// JS LOADER
|
||||
// Reference: https://github.com/babel/babel-loader
|
||||
// Transpile .js files using babel-loader
|
||||
|
|
@ -231,7 +237,8 @@ module.exports = function makeWebpackConfig () {
|
|||
'process.env': {
|
||||
HELIUM_BUNDLE_DEV: process.env.HELIUM_BUNDLE_DEV,
|
||||
SERVER_PORT: serverPort,
|
||||
WEB_PORT: webPort
|
||||
WEB_PORT: webPort,
|
||||
BUILD_CI: (isCI) ? JSON.stringify(true) : JSON.stringify(false)
|
||||
}
|
||||
})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -901,14 +901,18 @@ public class InterpreterSettingManager {
|
|||
saveToFile();
|
||||
}
|
||||
|
||||
public void removeNoteInterpreterSettingBinding(String user, String noteId) {
|
||||
public void removeNoteInterpreterSettingBinding(String user, String noteId) throws IOException {
|
||||
synchronized (interpreterSettings) {
|
||||
List<String> settingIds = (interpreterBindings.containsKey(noteId) ?
|
||||
interpreterBindings.remove(noteId) :
|
||||
Collections.<String>emptyList());
|
||||
for (String settingId : settingIds) {
|
||||
this.removeInterpretersForNote(get(settingId), user, noteId);
|
||||
InterpreterSetting setting = get(settingId);
|
||||
if (setting != null) {
|
||||
this.removeInterpretersForNote(setting, user, noteId);
|
||||
}
|
||||
}
|
||||
saveToFile();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -338,7 +338,11 @@ public class Notebook implements NoteEventListener {
|
|||
note = notes.remove(id);
|
||||
folders.removeNote(note);
|
||||
}
|
||||
interpreterSettingManager.removeNoteInterpreterSettingBinding(subject.getUser(), id);
|
||||
try {
|
||||
interpreterSettingManager.removeNoteInterpreterSettingBinding(subject.getUser(), id);
|
||||
} catch (IOException e) {
|
||||
logger.error(e.toString(), e);
|
||||
}
|
||||
noteSearchService.deleteIndexDocs(note);
|
||||
notebookAuthorization.removeNote(id);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue