Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Make ZeppelinSparkClusterTest.java work with spark 2

Browse source
This commit is contained in:
Lee moon soo 2016-07-17 04:13:45 +09:00
parent 839912a6ab
commit 862ff6c299
1 changed files with 17 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
View file

@ -136,16 +136,22 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
Note note = ZeppelinServer.notebook.createNote(null);
note.setName("note");
int sparkVersion = getSparkVersionNumber(note);
int sparkVersionNumber = getSparkVersionNumber(note);
if (isPyspark() && sparkVersion >= 14) { // auto_convert enabled from spark 1.4
if (isPyspark() && sparkVersionNumber >= 14) { // auto_convert enabled from spark 1.4
// run markdown paragraph, again
Paragraph p = note.addParagraph();
Map config = p.getConfig();
config.put("enabled", true);
p.setConfig(config);
String sqlContextName = "sqlContext";
if (sparkVersionNumber >= 20) {
sqlContextName = "spark";
}
p.setText("%pyspark\nfrom pyspark.sql.functions import *\n"
+ "print(sqlContext.range(0, 10).withColumn('uniform', rand(seed=10) * 3.14).count())");
+ "print(" + sqlContextName + ".range(0, 10).withColumn('uniform', rand(seed=10) * 3.14).count())");
// p.getRepl("org.apache.zeppelin.spark.SparkInterpreter").open();
note.run(p.getId());
waitForFinish(p);
@ -191,8 +197,9 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
public void pySparkDepLoaderTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(null);
int sparkVersionNumber = getSparkVersionNumber(note);
if (isPyspark() && getSparkVersionNumber(note) >= 14) {
if (isPyspark() && sparkVersionNumber >= 14) {
// restart spark interpreter
List<InterpreterSetting> settings =
ZeppelinServer.notebook.getBindedInterpreterSettings(note.id());
@ -221,9 +228,14 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
// load data using libraries from dep loader
Paragraph p1 = note.addParagraph();
p1.setConfig(config);
String sqlContextName = "sqlContext";
if (sparkVersionNumber >= 20) {
sqlContextName = "spark";
}
p1.setText("%pyspark\n" +
"from pyspark.sql import SQLContext\n" +
"print(sqlContext.read.format('com.databricks.spark.csv')" +
"print(" + sqlContextName + ".read.format('com.databricks.spark.csv')" +
".load('"+ tmpFile.getAbsolutePath() +"').count())");
note.run(p1.getId());