Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Java Interpreter first version

Browse source
This commit is contained in:
Vincenzo Selvaggio 2018-07-21 15:24:33 +01:00
parent df346d9f47
commit f542a4f618
12 changed files with 454 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
beam/pom.xml
View file

@ -227,7 +227,14 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-java</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>

70
beam/src/main/java/org/apache/zeppelin/beam/BeamInterpreter.java
View file

@ -17,83 +17,17 @@
package org.apache.zeppelin.beam;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.zeppelin.java.JavaInterpreter;
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
/**
* Beam interpreter
*/
public class BeamInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(BeamInterpreter.class);
public class BeamInterpreter extends JavaInterpreter {
public BeamInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
File dir = new File(".");
// delete all .class files created while compilation process
for (int i = 0; i < dir.list().length; i++) {
File f = dir.listFiles()[i];
if (f.getAbsolutePath().endsWith(".class")) {
f.delete();
}
}
}
@Override
public InterpreterResult interpret(String code, InterpreterContext context) {
// choosing new name to class containing Main method
String generatedClassName = "C" + UUID.randomUUID().toString().replace("-", "");
try {
String res = StaticRepl.execute(generatedClassName, code);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, res);
} catch (Exception e) {
logger.error("Exception in Interpreter while interpret", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return Collections.emptyList();
}
}

1
conf/interpreter-list
View file

@ -29,6 +29,7 @@ hbase org.apache.zeppelin:zeppelin-hbase:0.7.0 Hbase in
ignite org.apache.zeppelin:zeppelin-ignite_2.11:0.7.0 Ignite interpreter built with Scala 2.11
jdbc org.apache.zeppelin:zeppelin-jdbc:0.7.0 Jdbc interpreter
kylin org.apache.zeppelin:zeppelin-kylin:0.7.0 Kylin interpreter
java org.apache.zeppelin:zeppelin-java:0.7.0 Java interpreter
lens org.apache.zeppelin:zeppelin-lens:0.7.0 Lens interpreter
livy org.apache.zeppelin:zeppelin-livy:0.7.0 Livy interpreter
md org.apache.zeppelin:zeppelin-markdown:0.7.0 Markdown support

1
docs/_includes/themes/zeppelin/_navigation.html
View file

@ -136,6 +136,7 @@
<li><a href="{{BASE_PATH}}/interpreter/hdfs.html">HDFS</a></li>
<li><a href="{{BASE_PATH}}/interpreter/hive.html">Hive</a></li>
<li><a href="{{BASE_PATH}}/interpreter/ignite.html">Ignite</a></li>
<li><a href="{{BASE_PATH}}/interpreter/java.html">Java</a></li>
<li><a href="{{BASE_PATH}}/interpreter/kylin.html">Kylin</a></li>
<li><a href="{{BASE_PATH}}/interpreter/lens.html">Lens</a></li>
<li><a href="{{BASE_PATH}}/interpreter/livy.html">Livy</a></li>

113
docs/interpreter/java.md Normal file
View file

@ -0,0 +1,113 @@
---
layout: page
title: Java interpreter in Apache Zeppelin
description: Run Java code and any distributed java computation engine by importing the dependencies in the interpreter configuration.
group: interpreter
---
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
{% include JB/setup %}
# Java interpreter for Apache Zeppelin
<div id="toc"></div>
## How to use
Basically, you can write normal java code. You should write the main method inside a class because the interpreter invoke this main to execute the code. Unlike Zeppelin normal pattern, each paragraph is considered as a separate job, there isn't any relation to any other paragraph.
TODO: UPDATE EXAMPLE BELOW WITH A JAVA ONE... TRY IT IN THE INTERPRETER FIRST
The following is a demonstration of a word count example with data represented in array of strings
But it can read data from files by replacing `Create.of(SENTENCES).withCoder(StringUtf8Coder.of())` with `TextIO.Read.from("path/to/filename.txt")`
```java
%java
// most used imports
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.transforms.Create;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
import java.util.ArrayList;
import org.apache.beam.runners.direct.*;
import org.apache.beam.sdk.runners.*;
import org.apache.beam.sdk.options.*;
import org.apache.beam.runners.flink.*;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Count;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.options.PipelineOptions;
public class MinimalWordCount {
static List<String> s = new ArrayList<>();
static final String[] SENTENCES_ARRAY = new String[] {
"Hadoop is the Elephant King!",
"A yellow and elegant thing.",
"He never forgets",
"Useful data, or lets",
"An extraneous element cling!",
"A wonderful king is Hadoop.",
"The elephant plays well with Sqoop.",
"But what helps him to thrive",
"Are Impala, and Hive,",
"And HDFS in the group.",
"Hadoop is an elegant fellow.",
"An elephant gentle and mellow.",
"He never gets mad,",
"Or does anything bad,",
"Because, at his core, he is yellow",
};
static final List<String> SENTENCES = Arrays.asList(SENTENCES_ARRAY);
public static void main(String[] args) {
PipelineOptions options = PipelineOptionsFactory.create().as(PipelineOptions.class);
options.setRunner(FlinkRunner.class);
Pipeline p = Pipeline.create(options);
p.apply(Create.of(SENTENCES).withCoder(StringUtf8Coder.of()))
.apply("ExtractWords", ParDo.of(new DoFn<String, String>() {
@ProcessElement
public void processElement(ProcessContext c) {
for (String word : c.element().split("[^a-zA-Z']+")) {
if (!word.isEmpty()) {
c.output(word);
}
}
}
}))
.apply(Count.<String> perElement())
.apply("FormatResults", ParDo.of(new DoFn<KV<String, Long>, String>() {
@ProcessElement
public void processElement(DoFn<KV<String, Long>, String>.ProcessContext arg0)
throws Exception {
s.add("\n" + arg0.element().getKey() + "\t" + arg0.element().getValue());
}
}));
p.run();
System.out.println("%table word\tcount");
for (int i = 0; i < s.size(); i++) {
System.out.print(s.get(i));
}
}
}
```

22
java/README.md Normal file
View file

@ -0,0 +1,22 @@
# Overview
Java interpreter for Apache Zeppelin
# Architecture
Current interpreter implementation supports the static repl. It compiles the code in memory, execute it and redirect the output to zeppelin.
## Building the Java Interpreter
You have to first build the Java interpreter by enable the **java** profile as follows:
```
mvn clean package -Pjava -DskipTests -Pscala-2.10
```
### Technical overview
* Upon starting an interpreter, an instance of `JavaCompiler` is created.
* When the user runs commands with beam, the `JavaParser` go through the code to get a class that contains the main method.
* Then it replaces the class name with random class name to avoid overriding while compilation. It creates new out & err stream to get the data in new stream instead of the console, to redirect output to zeppelin.
* If there is any error during compilation, it can catch and redirect to zeppelin.

88
java/pom.xml Normal file
View file

@ -0,0 +1,88 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.9.0-SNAPSHOT</version>
<relativePath>../interpreter-parent</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-java</artifactId>
<packaging>jar</packaging>
<version>0.9.0-SNAPSHOT</version>
<name>Zeppelin: Java interpreter</name>
<properties>
<!--library versions-->
<interpreter.name>java</interpreter.name>
</properties>
<dependencies>
<dependency>
<groupId>com.thoughtworks.qdox</groupId>
<artifactId>qdox</artifactId>
<version>2.0-M3</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>false</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

99
java/src/main/java/org/apache/zeppelin/java/JavaInterpreter.java Normal file
View file

@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.java;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
/**
* Java interpreter
*/
public class JavaInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(JavaInterpreter.class);
public JavaInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
}
@Override
public void close() {
File dir = new File(".");
// delete all .class files created while compilation process
for (int i = 0; i < dir.list().length; i++) {
File f = dir.listFiles()[i];
if (f.getAbsolutePath().endsWith(".class")) {
f.delete();
}
}
}
@Override
public InterpreterResult interpret(String code, InterpreterContext context) {
// choosing new name to class containing Main method
String generatedClassName = "C" + UUID.randomUUID().toString().replace("-", "");
try {
String res = StaticRepl.execute(generatedClassName, code);
return new InterpreterResult(InterpreterResult.Code.SUCCESS, res);
} catch (Exception e) {
logger.error("Exception in Interpreter while interpret", e);
return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {
}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public List<InterpreterCompletion> completion(String buf, int cursor,
InterpreterContext interpreterContext) {
return Collections.emptyList();
}
}

2
beam/src/main/java/org/apache/zeppelin/beam/StaticRepl.java → java/src/main/java/org/apache/zeppelin/java/StaticRepl.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.beam;
package org.apache.zeppelin.java;
import com.thoughtworks.qdox.JavaProjectBuilder;
import com.thoughtworks.qdox.model.JavaClass;

14
java/src/main/resources/interpreter-setting.json Normal file
View file

@ -0,0 +1,14 @@
[
{
"group": "java",
"name": "java",
"className": "org.apache.zeppelin.java.JavaInterpreter",
"defaultInterpreter": true,
"properties": {
},
"editor": {
"language": "java",
"editOnDblClick": false
}
}
]

98
java/src/test/org/apache/zeppelin/java/JavaInterpreterTest.java Normal file
View file

@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.java;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Properties;
import static org.junit.Assert.assertEquals;
/**
* JavaInterpreterTest
*/
public class JavaInterpreterTest {
private static JavaInterpreter java;
private static InterpreterContext context;
@BeforeClass
public static void setUp() {
Properties p = new Properties();
java = new JavaInterpreter(p);
java.open();
context = InterpreterContext.builder().build();
}
@AfterClass
public static void tearDown() {
java.close();
}
@Test
public void testStaticRepl() {
StringWriter writer = new StringWriter();
PrintWriter out = new PrintWriter(writer);
out.println("public class HelloWorld {");
out.println(" public static void main(String args[]) {");
out.println(" System.out.println(\"This is in another java file\");");
out.println(" }");
out.println("}");
out.close();
InterpreterResult res = java.interpret(writer.toString(), context);
assertEquals(InterpreterResult.Code.SUCCESS, res.code());
}
@Test
public void testStaticReplWithoutMain() {
StringBuffer sourceCode = new StringBuffer();
sourceCode.append("package org.mdkt;\n");
sourceCode.append("public class HelloClass {\n");
sourceCode.append(" public String hello() { return \"hello\"; }");
sourceCode.append("}");
InterpreterResult res = java.interpret(sourceCode.toString(), context);
assertEquals(InterpreterResult.Code.ERROR, res.code());
}
@Test
public void testStaticReplWithSyntaxError() {
StringWriter writer = new StringWriter();
PrintWriter out = new PrintWriter(writer);
out.println("public class HelloWorld {");
out.println(" public static void main(String args[]) {");
out.println(" System.out.prin(\"This is in another java file\");");
out.println(" }");
out.println("}");
out.close();
InterpreterResult res = java.interpret(writer.toString(), context);
assertEquals(InterpreterResult.Code.ERROR, res.code());
}
}

7
pom.xml
View file

@ -789,6 +789,13 @@
</modules>
</profile>
<profile>
<id>java</id>
<modules>
<module>java</module>
</modules>
</profile>
<profile>
<id>examples</id>
<modules>