mirror of
https://github.com/apache/zeppelin
synced 2026-05-24 09:38:26 +00:00
Refactoring of the code
This commit is contained in:
mahmoudelgamal
parent
3a2bd85298
commit
26fc59bc53
6 changed files with 50 additions and 58 deletions
36
beam/pom.xml
36
beam/pom.xml
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
|
||||
|
||||
|
||||
<parent>
|
||||
<artifactId>zeppelin</artifactId>
|
||||
<groupId>org.apache.zeppelin</groupId>
|
||||
|
|
@ -15,22 +16,26 @@
|
|||
<artifactId>zeppelin-beam</artifactId>
|
||||
<version>0.7.0-SNAPSHOT</version>
|
||||
|
||||
|
||||
|
||||
<repositories>
|
||||
<repository>
|
||||
<id>apache-beam</id>
|
||||
<url>https://repository.apache.org/content/repositories/snapshots/org/apache/beam/</url>
|
||||
</repository>
|
||||
</repositories>
|
||||
|
||||
<dependencies>
|
||||
<!-- <dependency> <groupId>com.google.cloud.dataflow</groupId> <artifactId>google-cloud-dataflow-java-sdk-all</artifactId>
|
||||
<version>1.6.0</version> </dependency> <dependency> <groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-parent</artifactId> <version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<artifactId>beam-runners-parent</artifactId> <version>0.1.0-incubating</version>
|
||||
<type>pom</type> </dependency> <dependency> <groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId> <version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId> <version>0.1.0-incubating</version>
|
||||
</dependency> <dependency> <groupId>org.apache.beam</groupId> <artifactId>beam-parent</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version> <type>pom</type> </dependency>
|
||||
<version>0.1.0-incubating</version> <type>pom</type> </dependency>
|
||||
<dependency> <groupId>org.apache.beam</groupId> <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
|
||||
<version>0.2.0-incubating</version> </dependency> <dependency> <groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-core-java</artifactId> <version>0.2.0-incubating</version>
|
||||
</dependency> <dependency> <groupId>org.apache.beam</groupId> <artifactId>beam-sdks-java-core</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version> </dependency> -->
|
||||
<version>0.1.0-incubating</version> </dependency> -->
|
||||
|
||||
<!--
|
||||
<dependency>
|
||||
|
|
@ -187,14 +192,14 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-parent</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<version>0.1.0-incubating</version>
|
||||
<type>pom</type>
|
||||
</dependency>
|
||||
<!--
|
||||
<dependency>
|
||||
<groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-examples-java</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<version>0.1.0-incubating</version>
|
||||
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
|
|
@ -207,7 +212,7 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-core-java</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<version>0.1.0-incubating</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<artifactId>google-http-client-jackson2</artifactId>
|
||||
|
|
@ -218,13 +223,13 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-direct-java</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<version>0.1.0-incubating</version>
|
||||
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-flink_2.10</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<version>0.1.0-incubating</version>
|
||||
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
|
|
@ -241,7 +246,7 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-flink_2.10-examples</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<version>0.1.0-incubating</version>
|
||||
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
|
|
@ -254,7 +259,7 @@
|
|||
<dependency>
|
||||
<groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<version>0.1.0-incubating</version>
|
||||
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
|
|
@ -267,7 +272,7 @@
|
|||
|
||||
<groupId>org.apache.beam</groupId>
|
||||
<artifactId>beam-runners-spark</artifactId>
|
||||
<version>0.2.0-incubating-SNAPSHOT</version>
|
||||
<version>0.1.0-incubating</version>
|
||||
<type>jar</type>
|
||||
</dependency>
|
||||
|
||||
|
|
@ -289,14 +294,13 @@
|
|||
|
||||
-->
|
||||
|
||||
<!--
|
||||
|
||||
<dependency>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>zeppelin-interpreter</artifactId>
|
||||
<version>${project.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-exec</artifactId>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
package org.apache.zeppelin.beam;
|
||||
|
||||
|
||||
import java.io.File;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
|
|
@ -23,18 +22,10 @@ import com.google.gson.Gson;
|
|||
*/
|
||||
public class BeamInterpreter extends Interpreter {
|
||||
|
||||
private String host = "http://localhost:8001";
|
||||
private InterpreterContext context;
|
||||
|
||||
public BeamInterpreter(Properties property) {
|
||||
super(property);
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void open() {
|
||||
|
||||
|
|
@ -42,16 +33,22 @@ public class BeamInterpreter extends Interpreter {
|
|||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
File dir = new File(".");
|
||||
for (int i = 0; i < dir.list().length; i++) {
|
||||
File f = dir.listFiles()[i];
|
||||
System.out.println(f.getAbsolutePath());
|
||||
if (f.getAbsolutePath().contains(".class"))
|
||||
f.delete();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public InterpreterResult interpret(String st, InterpreterContext context) {
|
||||
|
||||
String uuid = "C" + UUID.randomUUID().toString().replace("-", "");
|
||||
String className = "C" + UUID.randomUUID().toString().replace("-", "");
|
||||
|
||||
try {
|
||||
String msg = CompileSourceInMemory.execute(uuid, st);
|
||||
String msg = CompileSourceInMemory.execute(className, st);
|
||||
return new InterpreterResult(InterpreterResult.Code.SUCCESS, msg);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
|
|
|
|||
|
|
@ -14,20 +14,17 @@ import com.thoughtworks.qdox.model.JavaClass;
|
|||
import com.thoughtworks.qdox.model.JavaSource;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.PrintStream;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringReader;
|
||||
import java.io.StringWriter;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author admin
|
||||
* @author Mahmoud
|
||||
*
|
||||
*/
|
||||
public class CompileSourceInMemory {
|
||||
|
|
@ -39,31 +36,25 @@ public class CompileSourceInMemory {
|
|||
JavaProjectBuilder builder = new JavaProjectBuilder();
|
||||
JavaSource src = builder.addSource(new StringReader(code));
|
||||
|
||||
// List<String> imports = src.getImports();
|
||||
// String importsString = "";
|
||||
//
|
||||
// for (int i = 0; i < imports.size(); i++) {
|
||||
// importsString += "import " + imports.get(i) + ";\n";
|
||||
// }
|
||||
|
||||
List<JavaClass> classes = src.getClasses();
|
||||
String classesSt = "";
|
||||
String classMain = "", classMainName = "";
|
||||
String classMainName = null;
|
||||
for (int i = 0; i < classes.size(); i++) {
|
||||
boolean hasMain = false;
|
||||
for (int j = 0; j < classes.get(i).getMethods().size(); j++) {
|
||||
if (classes.get(i).getMethods().get(j).getName().equals("main")) {
|
||||
classMainName = classes.get(i).getName();
|
||||
hasMain = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (hasMain == true) {
|
||||
classMain = classes.get(i).getCodeBlock() + "\n";
|
||||
classMainName = classes.get(i).getName();
|
||||
} else
|
||||
classesSt += classes.get(i).getCodeBlock() + "\n";
|
||||
if (hasMain == true)
|
||||
break;
|
||||
|
||||
}
|
||||
|
||||
if (classMainName == null)
|
||||
throw new Exception("There isn't any class containing Main method.");
|
||||
|
||||
code = code.replace(classMainName, className);
|
||||
|
||||
StringWriter writer = new StringWriter();
|
||||
|
|
@ -71,11 +62,11 @@ public class CompileSourceInMemory {
|
|||
|
||||
out.println(code);
|
||||
out.close();
|
||||
|
||||
|
||||
System.out.println(writer.toString());
|
||||
|
||||
JavaFileObject file = new JavaSourceFromString(className, writer.toString());
|
||||
|
||||
|
||||
Iterable<? extends JavaFileObject> compilationUnits = Arrays.asList(file);
|
||||
|
||||
ByteArrayOutputStream baosOut = new ByteArrayOutputStream();
|
||||
|
|
@ -100,11 +91,9 @@ public class CompileSourceInMemory {
|
|||
}
|
||||
if (success) {
|
||||
try {
|
||||
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[] { new File("").toURI()
|
||||
.toURL() });
|
||||
Class.forName(className, true, classLoader)
|
||||
.getDeclaredMethod("main", new Class[] { String[].class })
|
||||
.invoke(null, new Object[] { null });
|
||||
|
||||
Class.forName(className).getDeclaredMethod("main", new Class[] { String[].class })
|
||||
.invoke(null, new Object[] { null });
|
||||
|
||||
System.out.flush();
|
||||
System.err.flush();
|
||||
|
|
@ -112,8 +101,8 @@ public class CompileSourceInMemory {
|
|||
System.setOut(oldOut);
|
||||
System.setErr(oldErr);
|
||||
|
||||
classLoader.clearAssertionStatus();
|
||||
|
||||
|
||||
return baosOut.toString();
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace(newErr);
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ cassandra org.apache.zeppelin:zeppelin-cassandra_2.11:0.6.1 Cassandr
|
|||
elasticsearch org.apache.zeppelin:zeppelin-elasticsearch:0.6.1 Elasticsearch interpreter
|
||||
file org.apache.zeppelin:zeppelin-file:0.6.1 HDFS file interpreter
|
||||
flink org.apache.zeppelin:zeppelin-flink_2.11:0.6.1 Flink interpreter built with Scala 2.11
|
||||
beam org.apache.zeppelin:zeppelin-beam:0.6.1 Beam interpreter
|
||||
hbase org.apache.zeppelin:zeppelin-hbase:0.6.1 Hbase interpreter
|
||||
ignite org.apache.zeppelin:zeppelin-ignite_2.11:0.6.1 Ignite interpreter built with Scala 2.11
|
||||
jdbc org.apache.zeppelin:zeppelin-jdbc:0.6.1 Jdbc interpreter
|
||||
|
|
|
|||
|
|
@ -184,13 +184,13 @@
|
|||
|
||||
<property>
|
||||
<name>zeppelin.interpreters</name>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter</value>
|
||||
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter</value>
|
||||
<description>Comma separated interpreter configurations. First interpreter become a default</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>zeppelin.interpreter.group.order</name>
|
||||
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery</value>
|
||||
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam</value>
|
||||
<description></description>
|
||||
</property>
|
||||
|
||||
|
|
|
|||
|
|
@ -529,7 +529,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
|
|||
+ "org.apache.zeppelin.scalding.ScaldingInterpreter,"
|
||||
+ "org.apache.zeppelin.jdbc.JDBCInterpreter,"
|
||||
+ "org.apache.zeppelin.hbase.HbaseInterpreter,"
|
||||
+ "org.apache.zeppelin.bigquery.BigQueryInterpreter"),
|
||||
+ "org.apache.zeppelin.bigquery.BigQueryInterpreter",
|
||||
+ "org.apache.zeppelin.beam.BeamInterpreter"),
|
||||
ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"),
|
||||
ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
|
||||
ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
|
||||
|
|
@ -537,7 +538,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
|
|||
ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
|
||||
ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
|
||||
+ "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
|
||||
+ "scalding,jdbc,hbase,bigquery"),
|
||||
+ "scalding,jdbc,hbase,bigquery,beam"),
|
||||
ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
|
||||
ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
|
||||
// use specified notebook (id) as homescreen
|
||||
|
|
|
|||
Loading…
Reference in a new issue