Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Don't need so many interpreters

Browse source
This commit is contained in:
Rohit Choudhary 2016-02-22 19:19:13 -08:00
parent de2fd3cd60
commit b53fd8bfb7
3 changed files with 0 additions and 397 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

119
livy/src/main/java/org/apache/zeppelin/livy/LivyPySparkInterpreter.java
View file

@ -1,119 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivyPySparkInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivyPySparkInterpreter.class);
static {
Interpreter.register(
"pyspark",
"lspark",
LivyPySparkInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.build()
);
}
private Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivyPySparkInterpreter(Properties property) {
super(property);
userSessionMap = new HashMap<>();
livyHelper = new LivyHelper(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext.getAuthenticationInfo().getUser(),
"pyspark")
);
} catch (Exception e) {
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpret(line, interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivyPySparkInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivyPySparkInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

119
livy/src/main/java/org/apache/zeppelin/livy/LivySparkRInterpreter.java
View file

@ -1,119 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivySparkRInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkRInterpreter.class);
static {
Interpreter.register(
"r",
"lspark",
LivySparkRInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.build()
);
}
private Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivySparkRInterpreter(Properties property) {
super(property);
userSessionMap = new HashMap<>();
livyHelper = new LivyHelper(property);
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext.getAuthenticationInfo().getUser(),
"sparkr")
);
} catch (Exception e) {
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
return livyHelper.interpret(line, interpreterContext, userSessionMap);
} catch (Exception e) {
LOGGER.error("Exception in LivySparkRInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivySparkRInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

159
livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java
View file

@ -1,159 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.livy;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* Livy PySpark interpreter for Zeppelin.
*/
public class LivySparkSQLInterpreter extends Interpreter {
Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class);
static String DEFAULT_MAX_RESULT = "1000";
static {
Interpreter.register(
"sql",
"lspark",
LivySparkSQLInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("livy.spark.maxResult",
DEFAULT_MAX_RESULT,
"Max number of SparkSQL result to display.")
.build()
);
}
private Map<String, Integer> userSessionMap;
private LivyHelper livyHelper;
public LivySparkSQLInterpreter(Properties property) {
super(property);
livyHelper = new LivyHelper(property);
userSessionMap = LivySparkInterpreter.getUserSessionMap();
}
@Override
public void open() {
}
@Override
public void close() {
}
@Override
public InterpreterResult interpret(String line, InterpreterContext interpreterContext) {
try {
if (userSessionMap.get(interpreterContext.getAuthenticationInfo().getUser()) == null) {
try {
userSessionMap.put(
interpreterContext.getAuthenticationInfo().getUser(),
livyHelper.createSession(
interpreterContext.getAuthenticationInfo().getUser(),
"spark")
);
livyHelper.initializeSpark(interpreterContext, userSessionMap);
} catch (Exception e) {
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
if (line == null || line.trim().length() == 0) {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, "");
}
InterpreterResult res = livyHelper.interpret("sqlContext.sql(\"" + line + "\").show("
+ property.get("livy.spark.maxResult") + ")",
interpreterContext, userSessionMap);
if (res.code() == InterpreterResult.Code.SUCCESS) {
StringBuilder resMsg = new StringBuilder();
resMsg.append("%table ");
String[] rows = res.message().split("\n");
String[] headers = rows[1].split("\\|");
for (int head = 1; head < headers.length; head++) {
resMsg.append(headers[head].trim()).append("\t");
}
resMsg.append("\n");
if (rows[3].indexOf("+") == 0) {
} else {
for (int cols = 3; cols < rows.length - 1; cols++) {
String[] col = rows[cols].split("\\|");
for (int data = 1; data < col.length; data++) {
resMsg.append(col[data].trim()).append("\t");
}
resMsg.append("\n");
}
}
if (rows[rows.length - 1].indexOf("only") == 0) {
resMsg.append("<font color=red>" + rows[rows.length - 1] + ".</font>");
}
return new InterpreterResult(InterpreterResult.Code.SUCCESS,
resMsg.toString()
);
} else {
return res;
}
} catch (Exception e) {
LOGGER.error("Exception in LivySparkSQLInterpreter while interpret ", e);
return new InterpreterResult(InterpreterResult.Code.ERROR,
InterpreterUtils.getMostRelevantMessage(e));
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
LivySparkInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}