diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE new file mode 100644 index 0000000000..322b69121e --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE @@ -0,0 +1,24 @@ +### What is this PR for? +A few sentences describing the overall goals of the pull request's commits. +First time? Check out the contributing guide - https://github.com/apache/incubator-zeppelin/blob/master/CONTRIBUTING.md + + +### What type of PR is it? +[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring] + +### Todos +* [ ] - Task + +### What is the Jira issue? +* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/ +* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533] + +### How should this be tested? +Outline the steps to test the PR here. + +### Screenshots (if appropriate) + +### Questions: +* Does the licenses files need update? +* Is there breaking changes for older versions? +* Does this needs documentation? diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27c4ef6274..f0eb97c736 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,6 +17,7 @@ In order to make the review process easier, please follow this template when mak ``` ### What is this PR for? A few sentences describing the overall goals of the pull request's commits. +First time? Check out the contributing guide - https://github.com/apache/incubator-zeppelin/blob/master/CONTRIBUTING.md ### What type of PR is it? [Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring] @@ -24,7 +25,9 @@ A few sentences describing the overall goals of the pull request's commits. ### Todos * [ ] - Task -### Is there a relevant Jira issue? +### What is the Jira issue? +* Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN/ +* Put link here, and add [ZEPPELIN-*Jira number*] in PR title, eg. [ZEPPELIN-533] ### How should this be tested? Outline the steps to test the PR here. @@ -37,12 +40,6 @@ Outline the steps to test the PR here. * Does this needs documentation? ``` -You can also use this small bookmarklet tool to fill your Pull Request fields automatically: - -``` -javascript:(function() {var e = document.getElementById('pull_request_body');if (e) {e.value += '### What is this PR for?\nA few sentences describing the overall goals of the pull request\'s commits.\n\n### What type of PR is it?\n[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]\n\n### Todos\n* [ ] - Task\n\n### Is there a relevant Jira issue?\n\n### How should this be tested?\nOutline the steps to test the PR here.\n\n### Screenshots (if appropriate)\n\n### Questions:\n* Does the licenses files need update?\n* Is there breaking changes for older versions?\n* Does this needs documentation?';}})(); -``` - ## Testing a Pull Request You can also test and review a particular Pull Request. Here are two useful ways. diff --git a/tachyon/pom.xml b/alluxio/pom.xml similarity index 73% rename from tachyon/pom.xml rename to alluxio/pom.xml index adcc47d9ad..132a525bbd 100644 --- a/tachyon/pom.xml +++ b/alluxio/pom.xml @@ -27,14 +27,15 @@ org.apache.zeppelin - zeppelin-tachyon + zeppelin-alluxio jar 0.6.0-incubating-SNAPSHOT - Zeppelin: Tachyon interpreter + Zeppelin: Alluxio interpreter http://www.apache.org - 0.8.2 + 1.0.0 + 1.6.1 @@ -50,9 +51,9 @@ - org.tachyonproject - tachyon-shell - ${tachyon.version} + org.alluxio + alluxio-shell + ${alluxio.version} @@ -73,23 +74,58 @@ - org.tachyonproject - tachyon-servers - ${tachyon.version} + org.mockito + mockito-all + 1.10.8 test - org.tachyonproject - tachyon-minicluster - ${tachyon.version} + org.powermock + powermock-api-mockito + ${powermock.version} test - org.tachyonproject - tachyon-underfs-local - ${tachyon.version} + org.powermock + powermock-core + ${powermock.version} + test + + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + + org.powermock + powermock-reflect + ${powermock.version} + test + + + + org.alluxio + alluxio-core-server + ${alluxio.version} + test + + + + org.alluxio + alluxio-minicluster + ${alluxio.version} + test + + + + org.alluxio + alluxio-underfs-local + ${alluxio.version} test @@ -127,7 +163,7 @@ copy-dependencies - ${project.build.directory}/../../interpreter/tachyon + ${project.build.directory}/../../interpreter/alluxio false false true @@ -141,7 +177,7 @@ copy - ${project.build.directory}/../../interpreter/tachyon + ${project.build.directory}/../../interpreter/alluxio false false true diff --git a/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java b/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java new file mode 100644 index 0000000000..acc59bdcc8 --- /dev/null +++ b/alluxio/src/main/java/org/apache/zeppelin/alluxio/AlluxioInterpreter.java @@ -0,0 +1,254 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.alluxio; + +import java.io.IOException; +import java.io.PrintStream; +import java.io.ByteArrayOutputStream; +import java.util.*; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import alluxio.Configuration; +import alluxio.shell.AlluxioShell; + +/** + * Alluxio interpreter for Zeppelin. + */ +public class AlluxioInterpreter extends Interpreter { + + Logger logger = LoggerFactory.getLogger(AlluxioInterpreter.class); + + protected static final String ALLUXIO_MASTER_HOSTNAME = "alluxio.master.hostname"; + protected static final String ALLUXIO_MASTER_PORT = "alluxio.master.port"; + + private AlluxioShell fs; + + private int totalCommands = 0; + private int completedCommands = 0; + + private final String alluxioMasterHostname; + private final String alluxioMasterPort; + + protected final List keywords = Arrays.asList("cat", "chgrp", + "chmod", "chown", "copyFromLocal", "copyToLocal", "count", + "createLineage", "deleteLineage", "du", "fileInfo", "free", + "getCapacityBytes", "getUsedBytes", "listLineages", "load", + "loadMetadata", "location", "ls", "mkdir", "mount", "mv", + "persist", "pin", "report", "rm", "setTtl", "tail", "touch", + "unmount", "unpin", "unsetTtl"); + + public AlluxioInterpreter(Properties property) { + super(property); + + alluxioMasterHostname = property.getProperty(ALLUXIO_MASTER_HOSTNAME); + alluxioMasterPort = property.getProperty(ALLUXIO_MASTER_PORT); + } + + static { + Interpreter.register("alluxio", "alluxio", + AlluxioInterpreter.class.getName(), + new InterpreterPropertyBuilder() + .add(ALLUXIO_MASTER_HOSTNAME, "localhost", "Alluxio master hostname") + .add(ALLUXIO_MASTER_PORT, "19998", "Alluxio master port") + .build()); + } + + @Override + public void open() { + logger.info("Starting Alluxio shell to connect to " + alluxioMasterHostname + + " on port " + alluxioMasterPort); + + System.setProperty(ALLUXIO_MASTER_HOSTNAME, alluxioMasterHostname); + System.setProperty(ALLUXIO_MASTER_PORT, alluxioMasterPort); + fs = new AlluxioShell(new Configuration()); + } + + @Override + public void close() { + logger.info("Closing Alluxio shell"); + + try { + fs.close(); + } catch (IOException e) { + logger.error("Cannot close connection", e); + } + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + String[] lines = splitAndRemoveEmpty(st, "\n"); + return interpret(lines, context); + } + + private InterpreterResult interpret(String[] commands, InterpreterContext context) { + boolean isSuccess = true; + totalCommands = commands.length; + completedCommands = 0; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos); + PrintStream old = System.out; + + System.setOut(ps); + + for (String command : commands) { + int commandResuld = 1; + String[] args = splitAndRemoveEmpty(command, " "); + if (args.length > 0 && args[0].equals("help")) { + System.out.println(getCommandList()); + } else { + commandResuld = fs.run(args); + } + if (commandResuld != 0) { + isSuccess = false; + break; + } else { + completedCommands += 1; + } + System.out.println(); + } + + System.out.flush(); + System.setOut(old); + + if (isSuccess) { + return new InterpreterResult(Code.SUCCESS, baos.toString()); + } else { + return new InterpreterResult(Code.ERROR, baos.toString()); + } + } + + private String[] splitAndRemoveEmpty(String st, String splitSeparator) { + String[] voices = st.split(splitSeparator); + ArrayList result = new ArrayList(); + for (String voice : voices) { + if (!voice.trim().isEmpty()) { + result.add(voice); + } + } + return result.toArray(new String[result.size()]); + } + + private String[] splitAndRemoveEmpty(String[] sts, String splitSeparator) { + ArrayList result = new ArrayList(); + for (String st : sts) { + result.addAll(Arrays.asList(splitAndRemoveEmpty(st, splitSeparator))); + } + return result.toArray(new String[result.size()]); + } + + @Override + public void cancel(InterpreterContext context) { } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) { + return completedCommands * 100 / totalCommands; + } + + @Override + public List completion(String buf, int cursor) { + String[] words = splitAndRemoveEmpty(splitAndRemoveEmpty(buf, "\n"), " "); + String lastWord = ""; + if (words.length > 0) { + lastWord = words[ words.length - 1 ]; + } + ArrayList voices = new ArrayList(); + for (String command : keywords) { + if (command.startsWith(lastWord)) { + voices.add(command); + } + } + return voices; + } + + private String getCommandList() { + StringBuilder sb = new StringBuilder(); + sb.append("Commands list:"); + sb.append("\n\t[help] - List all available commands."); + sb.append("\n\t[cat ] - Prints the file's contents to the console."); + sb.append("\n\t[chgrp [-R] ] - Changes the group of a file or directory " + + "specified by args. Specify -R to change the group recursively."); + sb.append("\n\t[chmod -R ] - Changes the permission of a file or directory " + + "specified by args. Specify -R to change the permission recursively."); + sb.append("\n\t[chown -R ] - Changes the owner of a file or directory " + + "specified by args. Specify -R to change the owner recursively."); + sb.append("\n\t[copyFromLocal ] - Copies a file or a directory from " + + "local filesystem to Alluxio filesystem."); + sb.append("\n\t[copyToLocal ] - Copies a file or a directory from the " + + "Alluxio filesystem to the local filesystem."); + sb.append("\n\t[count ] - Displays the number of files and directories matching " + + "the specified prefix."); + sb.append("\n\t[createLineage " + + "[ ...]] - Creates a lineage."); + sb.append("\n\t[deleteLineage ] - Deletes a lineage. If " + + "cascade is specified as true, dependent lineages will also be deleted."); + sb.append("\n\t[du ] - Displays the size of the specified file or directory."); + sb.append("\n\t[fileInfo ] - Displays all block info for the specified file."); + sb.append("\n\t[free ] - Removes the file or directory(recursively) " + + "from Alluxio memory space."); + sb.append("\n\t[getCapacityBytes] - Gets the capacity of the Alluxio file system."); + sb.append("\n\t[getUsedBytes] - Gets number of bytes used in the Alluxio file system."); + sb.append("\n\t[listLineages] - Lists all lineages."); + sb.append("\n\t[load ] - Loads a file or directory in Alluxio space, makes it " + + "resident in memory."); + sb.append("\n\t[loadMetadata ] - Loads metadata for the given Alluxio path from the " + + "under file system."); + sb.append("\n\t[location ] - Displays the list of hosts storing the specified file."); + sb.append("\n\t[ls [-R] ] - Displays information for all files and directories " + + "directly under the specified path. Specify -R to display files and " + + "directories recursively."); + sb.append("\n\t[mkdir [path2] ... [pathn]] - Creates the specified directories, " + + "including any parent directories that are required."); + sb.append("\n\t[mount ] - Mounts a UFS path onto an Alluxio path."); + sb.append("\n\t[mv ] - Renames a file or directory."); + sb.append("\n\t[persist ] - Persists a file or directory currently stored " + + "only in Alluxio to the UnderFileSystem."); + sb.append("\n\t[pin ] - Pins the given file or directory in memory (works " + + "recursively for directories). Pinned files are never evicted from memory, unless " + + "TTL is set."); + sb.append("\n\t[report ] - Reports to the master that a file is lost."); + sb.append("\n\t[rm [-R] ] - Removes the specified file. Specify -R to remove file or " + + "directory recursively."); + sb.append("\n\t[setTtl