Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ZEPPELIN-2176 groovy interpreter, fix unchecked, add to configs, move HTTP.groovy to resources to simplify build, add default z-properties

Browse source
This commit is contained in:
dlukyanov 2017-03-19 09:48:53 +02:00
parent aa427cdd88
commit 5fa26e00c3
10 changed files with 209 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
conf/zeppelin-site.xml.template
View file

@ -247,13 +247,13 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.python.PythonInterpreterPandasSql,org.apache.zeppelin.python.PythonCondaInterpreter,org.apache.zeppelin.python.PythonDockerInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivyPySpark3Interpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter,org.apache.zeppelin.pig.PigInterpreter,org.apache.zeppelin.pig.PigQueryInterpreter,org.apache.zeppelin.scio.ScioInterpreter,org.apache.zeppelin.groovy.GroovyInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>
<property>
<name>zeppelin.interpreter.group.order</name>
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam</value>
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam,groovy</value>
<description></description>
</property>

166
groovy/pom-groovy-only.xml Normal file
View file

@ -0,0 +1,166 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<!--version>0.6.2</version-->
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-groovy</artifactId>
<packaging>jar</packaging>
<version>0.8.0-SNAPSHOT</version>
<name>Zeppelin: Groovy interpreter</name>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/org.codehaus.groovy/groovy-all -->
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
<version>2.4.7</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<showDeprecation>true</showDeprecation>
<compilerArgs>
<!--arg>-verbose</arg-->
<arg>-Xlint:unchecked</arg>
</compilerArgs>
</configuration>
</plugin>
<!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
<executions>
</executions>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/groovy</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<!--this one only for independent groovy interpreter assembly-->
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.5.3</version>
<configuration>
<descriptor>src/assembly/dep.xml</descriptor>
</configuration>
<executions>
<execution>
<id>create-archive</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

31
groovy/pom.xml
View file

@ -22,7 +22,7 @@
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<!--version>0.7</version-->
<!--version>0.6.2</version-->
<version>0.8.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
@ -67,6 +67,19 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<showDeprecation>true</showDeprecation>
<compilerArgs>
<!--arg>-verbose</arg-->
<arg>-Xlint:unchecked</arg>
</compilerArgs>
</configuration>
</plugin>
<!--TODO: comment local `maven-checkstyle-plugin` and use zeppelin common check style-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -130,22 +143,6 @@
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.5.3</version>
<configuration>
<descriptor>src/assembly/dep.xml</descriptor>
</configuration>
<executions>
<execution>
<id>create-archive</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

6
groovy/src/assembly/dep.xml
View file

@ -40,13 +40,13 @@
<include>revision.txt</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.basedir}/src/main/groovy/</directory>
<!--fileSet>
<directory>${project.basedir}/src/main/groovy/classes/</directory>
<outputDirectory>/classes/</outputDirectory>
<includes>
<include>*.groovy</include>
</includes>
</fileSet>
</fileSet-->
<!--fileSet>
<directory>${project.build.directory}/site</directory>

2
groovy/src/main/groovy/readme.txt
View file

@ -1,2 +0,0 @@
folder to place some groovy helpers to simplify custom code
as an example there is an HTTP.goovy simple class to call http/rest services

3
groovy/src/main/java/org/apache/zeppelin/groovy/GObject.java
View file

@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.groovy;
import java.io.StringWriter;
import org.slf4j.Logger;
import java.util.Properties;
@ -145,6 +145,7 @@ public class GObject extends groovy.lang.GroovyObjectSupport {
}
}
@SuppressWarnings("unchecked")
public void angularBind(String name, Object o, String noteId) {
AngularObjectRegistry registry = interpreterContext.getAngularObjectRegistry();

17
groovy/src/main/java/org/apache/zeppelin/groovy/GroovyInterpreter.java
View file

@ -75,7 +75,9 @@ public class GroovyInterpreter extends Interpreter {
}catch(Exception e){}
}
log.info("groovy classes classpath: "+classes);
if(classes!=null && classes.length()>0){
if(classes!=null && classes.length()>0 ){
File fClasses = new File(classes);
if(!fClasses.exists())fClasses.mkdirs();
shell.getClassLoader().addClasspath(classes);
}
}
@ -116,7 +118,10 @@ public class GroovyInterpreter extends Interpreter {
return null;
}
Map<String,Class<Script>> scriptCache = Collections.synchronizedMap( new WeakHashMap(1000) );
//cache for groovy compiled scripts
Map<String,Class<Script>> scriptCache = Collections.synchronizedMap( new WeakHashMap<String,Class<Script>>(1000) );
@SuppressWarnings("unchecked")
Script getGroovyScript(String id, String scriptText) /*throws SQLException*/ {
if(shell==null){
throw new RuntimeException("Groovy Shell is not initialized: null");
@ -138,12 +143,13 @@ public class GroovyInterpreter extends Interpreter {
@Override
@SuppressWarnings("unchecked")
public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
try {
Script script = getGroovyScript(contextInterpreter.getParagraphId(), cmd);
Job runningJob = getRunningJob(contextInterpreter.getParagraphId());
runningJob.info().put("CURRENT_THREAD", Thread.currentThread()); //to be able to terminate thread
Map bindings = script.getBinding().getVariables();
Map<String,Object> bindings = script.getBinding().getVariables();
bindings.clear();
StringWriter out = new StringWriter( (int) (cmd.length()*1.75) );
@ -152,7 +158,7 @@ public class GroovyInterpreter extends Interpreter {
script.run();
bindings.clear();
InterpreterResult result = new InterpreterResult(Code.SUCCESS, out.toString());
log.info("RESULT: "+result);
//log.info("RESULT: "+result);
return result;
}catch(Throwable t){
t = StackTraceUtils.deepSanitize(t);
@ -173,7 +179,8 @@ public class GroovyInterpreter extends Interpreter {
try {
Thread t = (Thread) object;
t.dumpStack();
t.stop();
t.interrupt();
//t.stop();
}catch(Throwable t){
log.error("Failed to cancel script: "+t, t);
}

0
groovy/src/main/groovy/HTTP.groovy → groovy/src/main/resources/HTTP.groovy
View file

6
groovy/src/main/resources/interpreter-setting.json
View file

@ -4,6 +4,12 @@
"name": "groovy",
"className": "org.apache.zeppelin.groovy.GroovyInterpreter",
"properties": {
"GROOVY_CLASSES": {
"envName": null,
"propertyName": "GROOVY_CLASSES",
"defaultValue": "",
"description": "The path for custom groovy classes location. If empty `./interpreter/groovy/classes`"
}
}
}
]

6
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -585,7 +585,9 @@ public class ZeppelinConfiguration extends XMLConfiguration {
+ "org.apache.zeppelin.hbase.HbaseInterpreter,"
+ "org.apache.zeppelin.bigquery.BigQueryInterpreter,"
+ "org.apache.zeppelin.beam.BeamInterpreter,"
+ "org.apache.zeppelin.scio.ScioInterpreter"),
+ "org.apache.zeppelin.scio.ScioInterpreter,"
+ "org.apache.zeppelin.groovy.GroovyInterpreter"
),
ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"),
ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
@ -593,7 +595,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
+ "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
+ "scalding,jdbc,hbase,bigquery,beam,pig,scio"),
+ "scalding,jdbc,hbase,bigquery,beam,pig,scio","groovy"),
ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit", 1024 * 100),
ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),