Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Hbase Interpreter

Browse source
This commit is contained in:
Rajat Venkatesh 2015-04-29 10:10:45 +05:30
parent a0428f06d5
commit bdba5b25f4
6 changed files with 569 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
conf/zeppelin-site.xml.template
View file

@ -66,7 +66,7 @@
<property>
<name>zeppelin.interpreters</name>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter</value>
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.hive.HiveInterpreter,org.apache.zeppelin.tajo.TajoInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter</value>
<description>Comma separated interpreter configurations. First interpreter become a default</description>
</property>

173
hbase/pom.xml Normal file
View file

@ -0,0 +1,173 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>zeppelin</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.5.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<name>Zeppelin: Hbase interpreter</name>
<artifactId>zeppelin-hbase</artifactId>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>zeppelin-interpreter</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jruby</groupId>
<artifactId>jruby-complete</artifactId>
<version>1.6.8</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
<version>2.6.0</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-annotations</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
<version>2.12.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.7</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.3.1</version>
<executions>
<execution>
<id>enforce</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/hbase</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/../../interpreter/hbase</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
<artifactId>${project.artifactId}</artifactId>
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

130
hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java Normal file
View file

@ -0,0 +1,130 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.hbase;
import org.apache.commons.exec.*;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.jruby.embed.LocalContextScope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.jruby.embed.ScriptingContainer;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
/**
* Created by rajatv on 4/22/15.
*/
public class HbaseInterpreter extends Interpreter {
Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class);
int commandTimeOut = 600000;
ScriptingContainer scriptingContainer;
StringWriter writer;
static {
Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase")
.add("hbase.ruby.sources", "lib/ruby",
"Path to Ruby scripts relative to 'hbase.home'")
.add("hbase.irb.load", "true", "Load hirb. Optional for testing only")
.build());
}
public HbaseInterpreter(Properties property) {
super(property);
}
@Override
public void open() {
String hbase_home = getProperty("hbase.home");
String ruby_src = getProperty("hbase.ruby.sources");
String abs_ruby_src = hbase_home + ruby_src;
logger.info("Home:" + hbase_home);
logger.info("Ruby Src:" + ruby_src);
Properties props = System.getProperties();
props.setProperty("hbase.ruby.sources", abs_ruby_src);
this.scriptingContainer = new ScriptingContainer(LocalContextScope.SINGLETON);
List<String> paths = new ArrayList<>(Arrays.asList(abs_ruby_src));
this.writer = new StringWriter();
scriptingContainer.setOutput(this.writer);
this.scriptingContainer.setLoadPaths(paths);
scriptingContainer.setCompatVersion(org.jruby.CompatVersion.RUBY1_9);
if (Boolean.parseBoolean(getProperty("hbase.irb.load"))) {
try {
InputStream in = getClass().getResourceAsStream("/hbase/bin/hirb.rb");
scriptingContainer.runScriptlet(in, "/hbase/bin/hirb.rb");
in.close();
} catch (NullPointerException | IOException e) {
logger.error("Open failed:", e);
}
}
}
@Override
public void close() {}
@Override
public InterpreterResult interpret(String cmd, InterpreterContext interpreterContext) {
try {
logger.info(cmd);
this.writer.getBuffer().setLength(0);
this.scriptingContainer.runScriptlet(cmd);
this.writer.flush();
logger.debug(writer.toString());
return new InterpreterResult(InterpreterResult.Code.SUCCESS, writer.getBuffer().toString());
} catch (Throwable t) {
logger.error("Can not run " + cmd, t);
return new InterpreterResult(InterpreterResult.Code.ERROR, t.getMessage());
}
}
@Override
public void cancel(InterpreterContext context) {}
@Override
public FormType getFormType() {
return FormType.SIMPLE;
}
@Override
public int getProgress(InterpreterContext context) {
return 0;
}
@Override
public Scheduler getScheduler() {
return SchedulerFactory.singleton().createOrGetFIFOScheduler(
HbaseInterpreter.class.getName() + this.hashCode());
}
@Override
public List<String> completion(String buf, int cursor) {
return null;
}
}

189
hbase/src/main/resources/hbase/bin/hirb.rb Normal file
View file

@ -0,0 +1,189 @@
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# File passed to org.jruby.Main by bin/hbase. Pollutes jirb with hbase imports
# and hbase commands and then loads jirb. Outputs a banner that tells user
# where to find help, shell version, and loads up a custom hirb.
# TODO: Interrupt a table creation or a connection to a bad master. Currently
# has to time out. Below we've set down the retries for rpc and hbase but
# still can be annoying (And there seem to be times when we'll retry for
# ever regardless)
# TODO: Add support for listing and manipulating catalog tables, etc.
# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes
# Run the java magic include and import basic HBase types that will help ease
# hbase hacking.
include Java
# Some goodies for hirb. Should these be left up to the user's discretion?
require 'irb/completion'
#
# FIXME: Switch args processing to getopt
#
# See if there are args for this shell. If any, read and then strip from ARGV
# so they don't go through to irb. Output shell 'usage' if user types '--help'
cmdline_help = <<HERE # HERE document output as shell usage
Usage: shell [OPTIONS] [SCRIPTFILE [ARGUMENTS]]
--format=OPTION Formatter for outputting results.
Valid options are: console, html.
(Default: console)
-d | --debug Set DEBUG log levels.
-h | --help This help.
HERE
found = []
format = 'console'
script2run = nil
log_level = org.apache.log4j.Level::ERROR
@shell_debug = false
for arg in ARGV
if arg =~ /^--format=(.+)/i
format = $1
if format =~ /^html$/i
raise NoMethodError.new("Not yet implemented")
elsif format =~ /^console$/i
# This is default
else
raise ArgumentError.new("Unsupported format " + arg)
end
found.push(arg)
elsif arg == '-h' || arg == '--help'
puts cmdline_help
exit
elsif arg == '-d' || arg == '--debug'
log_level = org.apache.log4j.Level::DEBUG
$fullBackTrace = true
@shell_debug = true
found.push(arg)
puts "Setting DEBUG log level..."
else
# Presume it a script. Save it off for running later below
# after we've set up some environment.
script2run = arg
found.push(arg)
# Presume that any other args are meant for the script.
break
end
end
# Delete all processed args
found.each { |arg| ARGV.delete(arg) }
# Make sure debug flag gets back to IRB
if @shell_debug
ARGV.unshift('-d')
end
# Set logging level to avoid verboseness
org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level)
org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level)
# Require HBase now after setting log levels
require 'hbase'
# Load hbase shell
require 'shell'
# Require formatter
require 'shell/formatter'
# Presume console format.
# Formatter takes an :output_stream parameter, if you don't want STDOUT.
@formatter = Shell::Formatter::Console.new
# Setup the HBase module. Create a configuration.
@hbase = Hbase::Hbase.new
# Setup console
@shell = Shell::Shell.new(@hbase, @formatter)
@shell.debug = @shell_debug
# Add commands to this namespace
@shell.export_commands(self)
# Add help command
def help(command = nil)
@shell.help(command)
end
# Backwards compatibility method
def tools
@shell.help_group('tools')
end
# Debugging method
def debug
if @shell_debug
@shell_debug = false
conf.back_trace_limit = 0
log_level = org.apache.log4j.Level::ERROR
else
@shell_debug = true
conf.back_trace_limit = 100
log_level = org.apache.log4j.Level::DEBUG
end
org.apache.log4j.Logger.getLogger("org.apache.zookeeper").setLevel(log_level)
org.apache.log4j.Logger.getLogger("org.apache.hadoop.hbase").setLevel(log_level)
debug?
end
def debug?
puts "Debug mode is #{@shell_debug ? 'ON' : 'OFF'}\n\n"
nil
end
# Include hbase constants
include HBaseConstants
# If script2run, try running it. Will go on to run the shell unless
# script calls 'exit' or 'exit 0' or 'exit errcode'.
load(script2run) if script2run
# Output a banner message that tells users where to go for help
@shell.print_banner
require "irb"
require 'irb/hirb'
module IRB
def self.start(ap_path = nil)
$0 = File::basename(ap_path, ".rb") if ap_path
IRB.setup(ap_path)
@CONF[:IRB_NAME] = 'hbase'
@CONF[:AP_NAME] = 'hbase'
@CONF[:BACK_TRACE_LIMIT] = 0 unless $fullBackTrace
if @CONF[:SCRIPT]
hirb = HIRB.new(nil, @CONF[:SCRIPT])
else
hirb = HIRB.new
end
@CONF[:IRB_RC].call(hirb.context) if @CONF[:IRB_RC]
@CONF[:MAIN_CONTEXT] = hirb.context
catch(:IRB_EXIT) do
hirb.eval_input
end
end
end
IRB.start

75
hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java Normal file
View file

@ -0,0 +1,75 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.hbase;
import org.apache.log4j.BasicConfigurator;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Properties;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.assertEquals;
/**
* Created by rajatv on 4/24/15.
*/
public class HbaseInterpreterTest {
private static Logger logger = LoggerFactory.getLogger(HbaseInterpreterTest.class);
private static HbaseInterpreter hbaseInterpreter;
@BeforeClass
public static void setUp() throws NullPointerException {
BasicConfigurator.configure();
Properties properties = new Properties();
properties.put("hbase.home", HbaseInterpreterTest.class.getClassLoader().getResource("ruby/").toString());
properties.put("hbase.ruby.sources", "");
properties.put("hbase.irb.load", "false");
logger.info("Resource: " + properties.getProperty("hbase.home"));
hbaseInterpreter = new HbaseInterpreter(properties);
hbaseInterpreter.open();
}
@Test
public void newObject() {
assertThat(hbaseInterpreter, notNullValue());
}
@Test
public void putsTest() {
InterpreterResult result = hbaseInterpreter.interpret("puts \"Hello World\"", null);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(result.type(), InterpreterResult.Type.TEXT);
assertEquals("Hello World\n", result.message());
}
public void putsLoadPath() {
InterpreterResult result = hbaseInterpreter.interpret("require 'two_power'; puts twoToThePowerOf(4)", null);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
assertEquals(result.type(), InterpreterResult.Type.TEXT);
assertEquals("16\n", result.message());
}
@Test
public void testException() {
InterpreterResult result = hbaseInterpreter.interpret("plot practical joke", null);
assertEquals(InterpreterResult.Code.ERROR, result.code());
assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message());
}
}

1
pom.xml
View file

@ -91,6 +91,7 @@
<module>angular</module>
<module>shell</module>
<module>hive</module>
<module>hbase</module>
<module>tajo</module>
<module>zeppelin-web</module>
<module>zeppelin-server</module>