Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Adding the beam to zeppelin 7

Browse source
This commit is contained in:
mahmoudelgamal 2016-07-20 16:26:49 +02:00
parent ab7ee2d837
commit 3a2bd85298
4 changed files with 19 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
beam/pom.xml
View file

@ -7,23 +7,16 @@
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.6.0-incubating-SNAPSHOT</version>
<version>0.7.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-beam</artifactId>
<version>0.6.0-incubating-SNAPSHOT</version>
<version>0.7.0-SNAPSHOT</version>
<repositories>
<repository>
<id>repo.bodar.com</id>
<url>http://repo.bodar.com</url>
</repository>
</repositories>
<dependencies>
<!-- <dependency> <groupId>com.google.cloud.dataflow</groupId> <artifactId>google-cloud-dataflow-java-sdk-all</artifactId>
@ -190,11 +183,6 @@
<version>2.12</version>
</dependency>
<dependency>
<groupId>com.googlecode.totallylazy</groupId>
<artifactId>totallylazy</artifactId>
<version>1.83</version>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
@ -300,14 +288,15 @@
</dependency>
-->
<!--
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
-->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
@ -327,26 +316,6 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.apache.beam.runners.flink.examples.WordCount</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>

8
beam/src/main/java/org/apache/zeppelin/beam/BeamInterpreter.java
View file

@ -10,11 +10,11 @@ import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import org.apache.beam.examples.MinimalWordCount;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import com.google.gson.Gson;
@ -30,10 +30,6 @@ public class BeamInterpreter extends Interpreter {
super(property);
}
static {
Interpreter.register("beam", "beam", BeamInterpreter.class.getName(),
new InterpreterPropertyBuilder().build());
}
public static void main(String[] args) {
@ -81,7 +77,7 @@ public class BeamInterpreter extends Interpreter {
}
@Override
public List<String> completion(String buf, int cursor) {
public List<InterpreterCompletion> completion(String buf, int cursor) {
return null;
}

27
beam/src/main/java/org/apache/zeppelin/beam/CompileSourceInMemory.java
View file

@ -8,8 +8,6 @@ import javax.tools.JavaFileObject;
import javax.tools.SimpleJavaFileObject;
import javax.tools.ToolProvider;
import org.apache.log4j.Category;
import org.apache.log4j.Priority;
import com.thoughtworks.qdox.JavaProjectBuilder;
import com.thoughtworks.qdox.model.JavaClass;
@ -17,7 +15,6 @@ import com.thoughtworks.qdox.model.JavaSource;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringReader;
@ -35,20 +32,20 @@ import java.util.List;
*/
public class CompileSourceInMemory {
public static String execute(String className, String code) throws Exception {
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<JavaFileObject>();
JavaProjectBuilder builder = new JavaProjectBuilder();
JavaSource src = builder.addSource(new StringReader(code));
// List<String> imports = src.getImports();
// String importsString = "";
//
// for (int i = 0; i < imports.size(); i++) {
// importsString += "import " + imports.get(i) + ";\n";
// }
// List<String> imports = src.getImports();
// String importsString = "";
//
// for (int i = 0; i < imports.size(); i++) {
// importsString += "import " + imports.get(i) + ";\n";
// }
List<JavaClass> classes = src.getClasses();
String classesSt = "";
String classMain = "", classMainName = "";
@ -80,10 +77,10 @@ public class CompileSourceInMemory {
JavaFileObject file = new JavaSourceFromString(className, writer.toString());
Iterable<? extends JavaFileObject> compilationUnits = Arrays.asList(file);
ByteArrayOutputStream baosOut = new ByteArrayOutputStream();
ByteArrayOutputStream baosErr = new ByteArrayOutputStream();
PrintStream newOut = new PrintStream(baosOut);
PrintStream newErr = new PrintStream(baosErr);
// IMPORTANT: Save the old System.out!
@ -93,7 +90,6 @@ public class CompileSourceInMemory {
System.setOut(newOut);
System.setErr(newErr);
CompilationTask task = compiler.getTask(null, null, diagnostics, null, null, compilationUnits);
boolean success = task.call();
@ -112,11 +108,10 @@ public class CompileSourceInMemory {
System.out.flush();
System.err.flush();
System.setOut(oldOut);
System.setErr(oldErr);
classLoader.clearAssertionStatus();
return baosOut.toString();

1
pom.xml
View file

@ -61,6 +61,7 @@
<module>angular</module>
<module>shell</module>
<module>livy</module>
<module>beam</module>
<module>hbase</module>
<module>postgresql</module>
<module>jdbc</module>