Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ZEPPELIN-3935. Enable color output of spark scala interpreter

Browse source
This commit is contained in:
jeffzhang.zjf 2019-01-24 17:07:49 +08:00 committed by Jeff Zhang
parent 29b301e1a3
commit d3c6f10b54
6 changed files with 35 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/interpreter/spark.md
View file

@ -160,6 +160,10 @@ You can also set other Spark properties which are not listed in the table. For a
<td></td>
<td>Overrides Spark UI default URL. Value should be a full URL (ex: http://{hostName}/{uniquePath}</td>
</tr>
<td>zeppelin.spark.scala.color</td>
<td>true</td>
<td>Whether to enable color output of spark scala interpreter</td>
</tr>
</table>
Without any configuration, Spark interpreter works out of box in local mode. But if you want to connect to your Spark cluster, you'll need to follow below two simple steps.

4
spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -45,6 +45,10 @@ public class SparkInterpreter extends AbstractSparkInterpreter {
public SparkInterpreter(Properties properties) {
super(properties);
// set scala.color
if (Boolean.parseBoolean(properties.getProperty("zeppelin.spark.scala.color", "true"))) {
System.setProperty("scala.color", "true");
}
if (Boolean.parseBoolean(properties.getProperty("zeppelin.spark.useNew", "false"))) {
delegation = new NewSparkInterpreter(properties);
} else {

7
spark/interpreter/src/main/resources/interpreter-setting.json
View file

@ -88,6 +88,13 @@
"defaultValue": false,
"description": "Whether to hide spark ui in zeppelin ui",
"type": "checkbox"
},
"zeppelin.spark.scala.color": {
"envName": null,
"propertyName": "zeppelin.spark.scala.color",
"defaultValue": true,
"description": "Whether to enable color output of spark scala interpreter",
"type": "checkbox"
}
},
"editor": {

16
spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
View file

@ -84,6 +84,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("zeppelin.spark.uiWebUrl", "fake_spark_weburl");
// disable color output for easy testing
properties.setProperty("zeppelin.spark.scala.color", "false");
InterpreterContext context = InterpreterContext.builder()
.setInterpreterOut(new InterpreterOutput(null))
@ -375,6 +377,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("spark.app.name", "test");
properties.setProperty("zeppelin.spark.maxResult", "100");
properties.setProperty("zeppelin.spark.useNew", "true");
// disable color output for easy testing
properties.setProperty("zeppelin.spark.scala.color", "false");
// download spark-avro jar
URL website = new URL("http://repo1.maven.org/maven2/com/databricks/spark-avro_2.11/3.2.0/spark-avro_2.11-3.2.0.jar");
@ -404,6 +408,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("zeppelin.dep.localrepo", Files.createTempDir().getAbsolutePath());
// disable color output for easy testing
properties.setProperty("zeppelin.spark.scala.color", "false");
InterpreterGroup intpGroup = new InterpreterGroup();
interpreter = new SparkInterpreter(properties);
@ -433,6 +439,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("zeppelin.spark.printREPLOutput", "false");
// disable color output for easy testing
properties.setProperty("zeppelin.spark.scala.color", "false");
InterpreterContext.set(getInterpreterContext());
interpreter = new SparkInterpreter(properties);
@ -460,6 +468,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("spark.scheduler.mode", "FAIR");
// disable color output for easy testing
properties.setProperty("zeppelin.spark.scala.color", "false");
interpreter = new SparkInterpreter(properties);
assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
@ -489,6 +499,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("spark.ui.enabled", "false");
// disable color output for easy testing
properties.setProperty("zeppelin.spark.scala.color", "false");
interpreter = new SparkInterpreter(properties);
assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
@ -514,6 +526,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("zeppelin.spark.ui.hidden", "true");
// disable color output for easy testing
properties.setProperty("zeppelin.spark.scala.color", "false");
interpreter = new SparkInterpreter(properties);
assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
@ -537,6 +551,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.maxResult", "100");
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
// disable color output for easy testing
properties.setProperty("zeppelin.spark.scala.color", "false");
SparkInterpreter interpreter1 = new SparkInterpreter(properties);
SparkInterpreter interpreter2 = new SparkInterpreter(properties);

2
spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java
View file

@ -85,6 +85,8 @@ public class OldSparkInterpreterTest {
p.setProperty("zeppelin.spark.importImplicit", "true");
p.setProperty("zeppelin.dep.localrepo", tmpDir.newFolder().getAbsolutePath());
p.setProperty("zeppelin.spark.property_1", "value_1");
// disable color output for easy testing
p.setProperty("zeppelin.spark.scala.color", "false");
return p;
}

2
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
View file

@ -122,6 +122,8 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
new InterpreterProperty("zeppelin.spark.test", "true"));
sparkProperties.put("spark.serializer",
new InterpreterProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer"));
sparkProperties.put("zeppelin.spark.scala.color",
new InterpreterProperty("zeppelin.spark.scala.color", "false"));
TestUtils.getInstance(Notebook.class).getInterpreterSettingManager().restart(sparkIntpSetting.getId());
}