From c66f90917955482e15db834566c242208dd361ba Mon Sep 17 00:00:00 2001 From: CloverHearts Date: Mon, 31 Jul 2017 19:29:36 +0900 Subject: [PATCH 01/82] [ZEPPELIN-2818] Improve to better rendering from jupyter note ### What is this PR for? Hi, zeppelin community. Zeppelin currently has a way to use the Jupiter Note. This is a great feature. However, I have found that there are some problems with the way of showing. I've improved this to increase the user experience. Please check the screenshot for details. ### What type of PR is it? Improvement ### Todos - [x] added pre-render feature (markdown). - [x] increased to many support type for output data. ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-2818 ### How should this be tested? 1. build jupyter module `mvn clean package -DskipTests -pl 'zeppelin-jupyter' --am` 2. `cd zeppelin-jupyter/target` 3. `java -classpath zeppelin-jupyter-0.8.0-SNAPSHOT.jar org.apache.zeppelin.jupyter.JupyterUtil -i {your ipynb note file path!/getting_started.ipynb` (good sample : [go to sample](https://github.com/SciRuby/sciruby-notebooks/blob/master/getting_started.ipynb) 4. get a `note.json` and import to zeppelin on frontend! 5. enjoy ### Screenshots (if appropriate) #### Before ![oldold](https://user-images.githubusercontent.com/10525473/28717881-8c176d40-73de-11e7-9a67-732808957391.gif) #### After ![zeppelin-from-jupyternote](https://user-images.githubusercontent.com/10525473/28717782-1fa90f4c-73de-11e7-8d9c-f5191d8f8a47.gif) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: CloverHearts Closes #2509 from cloverhearts/ZEPPELIN-2818 and squashes the following commits: 4f915739f [CloverHearts] added apache docuement c15bbcdb7 [CloverHearts] improving test case c8b5a86cc [CloverHearts] fix command line interpreter tag f4cb0037f [CloverHearts] support single line text e9076133a [CloverHearts] add setter f077e69ae [CloverHearts] added supported single line output c77d9c913 [CloverHearts] add supported single string for source 75c8ef540 [CloverHearts] import pegdown in pom b27bfc93f [CloverHearts] move interpreter to pegdown 1e15581d1 [CloverHearts] improving reder feature a56b6ffc6 [CloverHearts] fix checkstyle 4a5158603 [CloverHearts] implement markdown pre render 6ce9935b9 [CloverHearts] improving get output part --- zeppelin-jupyter/pom.xml | 10 ++ .../apache/zeppelin/jupyter/JupyterUtil.java | 98 +++++++----------- .../zeppelin/jupyter/nbformat/Cell.java | 4 +- .../jupyter/nbformat/DisplayData.java | 33 +++++++ .../zeppelin/jupyter/nbformat/Error.java | 17 ++++ .../jupyter/nbformat/ExecuteResult.java | 14 +++ .../zeppelin/jupyter/nbformat/Output.java | 99 +++++++++++++++++++ .../zeppelin/jupyter/nbformat/Stream.java | 35 ++++++- .../jupyter/types/JupyterOutputType.java | 70 +++++++++++++ .../jupyter/types/ZeppelinOutputType.java | 37 +++++++ .../zeppelin/jupyter/zformat/Paragraph.java | 25 +++++ .../zeppelin/jupyter/zformat/Result.java | 1 + .../jupyter/nbformat/JupyterUtilTest.java | 48 ++++++++- .../src/test/resources/basic.ipynb | 34 ++++++- 14 files changed, 453 insertions(+), 72 deletions(-) create mode 100644 zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/JupyterOutputType.java create mode 100644 zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/ZeppelinOutputType.java diff --git a/zeppelin-jupyter/pom.xml b/zeppelin-jupyter/pom.xml index 96dabd4d00..ae00db38d7 100644 --- a/zeppelin-jupyter/pom.xml +++ b/zeppelin-jupyter/pom.xml @@ -34,6 +34,10 @@ Zeppelin: Jupyter Support Jupyter support for Apache Zeppelin + + 1.6.0 + + com.google.code.gson @@ -55,6 +59,12 @@ commons-cli + + org.pegdown + pegdown + ${pegdown.version} + + junit diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java index 3879efb1b5..9bc5ca49fa 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java @@ -24,9 +24,8 @@ import java.io.Reader; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; -import java.util.Map; import com.google.common.base.Joiner; import com.google.gson.Gson; @@ -54,18 +53,18 @@ import org.apache.zeppelin.jupyter.zformat.Note; import org.apache.zeppelin.jupyter.zformat.Paragraph; import org.apache.zeppelin.jupyter.zformat.Result; import org.apache.zeppelin.jupyter.zformat.TypeData; +import org.pegdown.PegDownProcessor; /** * */ public class JupyterUtil { - private static final String TEXT_PLAIN = "text/plain"; - private static final String IMAGE_PNG = "image/png"; - private final RuntimeTypeAdapterFactory cellTypeFactory; private final RuntimeTypeAdapterFactory outputTypeFactory; + private final PegDownProcessor markdownProcessor; + public JupyterUtil() { this.cellTypeFactory = RuntimeTypeAdapterFactory.of(Cell.class, "cell_type") .registerSubtype(MarkdownCell.class, "markdown").registerSubtype(CodeCell.class, "code") @@ -74,6 +73,7 @@ public class JupyterUtil { .registerSubtype(ExecuteResult.class, "execute_result") .registerSubtype(DisplayData.class, "display_data").registerSubtype(Stream.class, "stream") .registerSubtype(Error.class, "error"); + this.markdownProcessor = new PegDownProcessor(); } public Nbformat getNbformat(Reader in) { @@ -103,67 +103,53 @@ public class JupyterUtil { note.setName(name); String lineSeparator = System.lineSeparator(); - String emptyString = ""; Paragraph paragraph; List paragraphs = new ArrayList<>(); String interpreterName; List typeDataList; - String type; - String result; for (Cell cell : nbformat.getCells()) { + String status = Result.SUCCESS; paragraph = new Paragraph(); typeDataList = new ArrayList<>(); + Object cellSource = cell.getSource(); + List sourceRaws = new ArrayList<>(); + + if (cellSource instanceof String) { + sourceRaws.add((String) cellSource); + } else { + sourceRaws.addAll((List) cellSource); + } + + List source = Output.verifyEndOfLine(sourceRaws); + String codeText = Joiner.on("").join(source); if (cell instanceof CodeCell) { interpreterName = codeReplaced; for (Output output : ((CodeCell) cell).getOutputs()) { - TypeData typeData; - if (output instanceof Stream) { - type = TypeData.TEXT; - List text = verifyEndOfLine(((Stream) output).getText(), lineSeparator); - result = Joiner.on(emptyString).join(text); - typeData = new TypeData(type, result); - typeDataList.add(typeData); - } else if (output instanceof ExecuteResult || output instanceof DisplayData) { - Map data = (output instanceof ExecuteResult) ? - ((ExecuteResult) output).getData() : - ((DisplayData) output).getData(); - for (Map.Entry datum : data.entrySet()) { - if (TEXT_PLAIN.equals(datum.getKey())) { - type = TypeData.TEXT; - List text = verifyEndOfLine((List) datum.getValue(), lineSeparator); - result = Joiner.on(emptyString).join(text); - } else if (IMAGE_PNG.equals(datum.getKey())) { - type = TypeData.HTML; - result = makeHTML(((String) datum.getValue()).replace("\n", "")); - } else { - type = TypeData.TEXT; - result = datum.getValue().toString(); - } - typeData = new TypeData(type, result); - typeDataList.add(typeData); - } + if (output instanceof Error) { + typeDataList.add(output.toZeppelinResult()); } else { - // Error - Error error = (Error) output; - type = TypeData.TEXT; - List text = verifyEndOfLine(Arrays.asList(error.getEname(), error.getEvalue()), - lineSeparator); - result = Joiner.on(emptyString).join(text); - typeData = new TypeData(type, result); - typeDataList.add(typeData); + typeDataList.add(output.toZeppelinResult()); + if (output instanceof Stream) { + Stream streamOutput = (Stream) output; + if (streamOutput.isError()) { + status = Result.ERROR; + } + } } } } else if (cell instanceof MarkdownCell || cell instanceof HeadingCell) { interpreterName = markdownReplaced; + String markdownContent = markdownProcessor.markdownToHtml(codeText); + typeDataList.add(new TypeData(TypeData.HTML, markdownContent)); + paragraph.setUpMarkdownConfig(true); } else { interpreterName = ""; } - List source = verifyEndOfLine(cell.getSource(), lineSeparator); - paragraph.setText(interpreterName + lineSeparator + Joiner.on(emptyString).join(source)); - paragraph.setResults(new Result(Result.SUCCESS, typeDataList)); + paragraph.setText(interpreterName + lineSeparator + codeText); + paragraph.setResults(new Result(status, typeDataList)); paragraphs.add(paragraph); } @@ -173,31 +159,13 @@ public class JupyterUtil { return note; } - private List verifyEndOfLine(List content, String lineSeparator) { - if (null == content || content.size() == 1) { - // one-liners don't have line separator - return content; - } - for (int i = 0; i < content.size(); i++) { - String line = content.get(i); - // verify to end with line separator except the last element - if (null != line && !line.endsWith(lineSeparator) && i != (content.size() - 1)) { - content.set(i, line + lineSeparator); - } - } - return content; - } + private Gson getGson(GsonBuilder gsonBuilder) { return gsonBuilder.registerTypeAdapterFactory(cellTypeFactory) .registerTypeAdapterFactory(outputTypeFactory).create(); } - private String makeHTML(String image) { - return "
"; - } - public static void main(String[] args) throws ParseException, IOException { Options options = new Options(); options.addOption("i", true, "Jupyter notebook file"); @@ -216,7 +184,7 @@ public class JupyterUtil { try (BufferedReader in = new BufferedReader(new FileReader(jupyterPath.toFile())); FileWriter fw = new FileWriter(zeppelinPath.toFile())) { - Note note = new JupyterUtil().getNote(in, "python", "md"); + Note note = new JupyterUtil().getNote(in, "%python", "%md"); Gson gson = new GsonBuilder().setPrettyPrinting().create(); gson.toJson(note, fw); } diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java index d44ec023f3..f593da51ca 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java @@ -31,7 +31,7 @@ public abstract class Cell { private CellMetadata metadata; @SerializedName("source") - private List source; + private Object source; public String getCellType() { return cellType; @@ -41,7 +41,7 @@ public abstract class Cell { return metadata; } - public List getSource() { + public Object getSource() { return source; } } diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java index 68bfbfeb85..6566e1d6fe 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java @@ -16,7 +16,14 @@ */ package org.apache.zeppelin.jupyter.nbformat; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; +import org.apache.zeppelin.jupyter.types.JupyterOutputType; +import org.apache.zeppelin.jupyter.types.ZeppelinOutputType; +import org.apache.zeppelin.jupyter.zformat.TypeData; + +import java.util.List; import java.util.Map; /** @@ -30,4 +37,30 @@ public class DisplayData extends Output { public Map getData() { return data; } + + @Override + public ZeppelinOutputType getTypeOfZeppelin() { + return getType(data).getZeppelinType(); + } + + @Override + public TypeData toZeppelinResult() { + return getZeppelinResult(data, getType(data)); + } + + private static class ZeppelinResultGenerator { + public static String toBase64ImageHtmlElement(String image) { + return "
"; + } + public static String toLatex(String latexCode) { + String latexContents = latexCode; + return "
" + + "
" + + "Warning! Currently, Latex is not supported." + + "
" + + "
" + latexContents + "
" + + "
"; + } + } } diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java index 61466fa7cb..679f4ccd33 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java @@ -16,7 +16,12 @@ */ package org.apache.zeppelin.jupyter.nbformat; +import com.google.common.base.Joiner; import com.google.gson.annotations.SerializedName; +import org.apache.zeppelin.jupyter.types.ZeppelinOutputType; +import org.apache.zeppelin.jupyter.zformat.TypeData; + +import java.util.Arrays; import java.util.List; /** @@ -43,4 +48,16 @@ public class Error extends Output { public List getTraceback() { return traceback; } + + @Override + public ZeppelinOutputType getTypeOfZeppelin() { + return ZeppelinOutputType.TEXT; + } + + @Override + public TypeData toZeppelinResult() { + List text = verifyEndOfLine(Arrays.asList(getEname(), getEvalue())); + String result = Joiner.on("").join(text); + return new TypeData(getTypeOfZeppelin().toString(), result); + } } diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java index 01f0cea047..9e8fe0562b 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java @@ -17,6 +17,10 @@ package org.apache.zeppelin.jupyter.nbformat; import com.google.gson.annotations.SerializedName; +import org.apache.zeppelin.jupyter.types.JupyterOutputType; +import org.apache.zeppelin.jupyter.types.ZeppelinOutputType; +import org.apache.zeppelin.jupyter.zformat.TypeData; + import java.util.Map; /** @@ -33,4 +37,14 @@ public class ExecuteResult extends Output { public Map getData() { return data; } + + @Override + public ZeppelinOutputType getTypeOfZeppelin() { + return getType(data).getZeppelinType(); + } + + @Override + public TypeData toZeppelinResult() { + return getZeppelinResult(data, getType(data)); + } } diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java index 50722d75f4..a37272a5b8 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java @@ -16,7 +16,15 @@ */ package org.apache.zeppelin.jupyter.nbformat; +import com.google.common.base.Joiner; import com.google.gson.annotations.SerializedName; +import org.apache.zeppelin.jupyter.types.JupyterOutputType; +import org.apache.zeppelin.jupyter.types.ZeppelinOutputType; +import org.apache.zeppelin.jupyter.zformat.TypeData; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; /** * @@ -25,4 +33,95 @@ public abstract class Output { @SerializedName("output_type") private String outputType; + + private static final transient String lineSeparator = System.lineSeparator(); + + public static List verifyEndOfLine(List content) { + if (null == content || content.size() == 1) { + // one-liners don't have line separator + return content; + } + for (int i = 0; i < content.size(); i++) { + String line = content.get(i); + // verify to end with line separator except the last element + if (null != line && !line.endsWith(lineSeparator) && i != (content.size() - 1)) { + content.set(i, line + lineSeparator); + } + } + return content; + } + + protected JupyterOutputType getType(Map data) { + JupyterOutputType jupyterOutputType = JupyterOutputType.TEXT_PLAIN; + + if (data == null) { + return null; + } + + for (String dataType : data.keySet()) { + if (!dataType.equals(JupyterOutputType.TEXT_PLAIN.toString())) { + try { + jupyterOutputType = JupyterOutputType.getByValue(dataType); + } catch (IllegalArgumentException e) { + // pass + } + } + } + + return jupyterOutputType; + } + + + protected TypeData getZeppelinResult(Map data, JupyterOutputType type) { + TypeData result = null; + Object outputsObject = data.get(type.toString()); + List outputsRaws = new ArrayList<>(); + if (outputsObject instanceof String) { + outputsRaws.add((String) outputsObject); + } else { + outputsRaws.addAll((List) outputsObject); + } + List outputs = verifyEndOfLine(outputsRaws); + String outputData = Joiner.on("").join(outputs); + if (type == JupyterOutputType.IMAGE_PNG) { + String base64CodeRaw = outputData; + String base64Code = base64CodeRaw.replace("\n", ""); + result = new TypeData( + type.getZeppelinType().toString(), + ZeppelinResultGenerator.toBase64ImageHtmlElement(base64Code) + ); + } else if (type == JupyterOutputType.LATEX) { + result = new TypeData( + type.getZeppelinType().toString(), + ZeppelinResultGenerator.toLatex(outputData) + ); + } else if (type == JupyterOutputType.APPLICATION_JAVASCRIPT) { + result = new TypeData( + type.getZeppelinType().toString(), + ZeppelinResultGenerator.toJavascript(outputData) + ); + } else { + result = new TypeData(type.getZeppelinType().toString(), outputData); + } + return result; + } + + public abstract ZeppelinOutputType getTypeOfZeppelin(); + public abstract TypeData toZeppelinResult(); + + private static class ZeppelinResultGenerator { + public static String toBase64ImageHtmlElement(String image) { + return "
"; + } + public static String toLatex(String latexCode) { + String latexContents = latexCode; + return "
" + + "
" + latexContents + "
" + + "
"; + } + public static String toJavascript(String javascriptCode) { + return ""; + } + } } diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java index a73f63550d..fa0e2465b3 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java @@ -16,7 +16,13 @@ */ package org.apache.zeppelin.jupyter.nbformat; +import com.google.common.base.Joiner; import com.google.gson.annotations.SerializedName; +import org.apache.zeppelin.jupyter.types.ZeppelinOutputType; +import org.apache.zeppelin.jupyter.zformat.TypeData; + +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -28,9 +34,34 @@ public class Stream extends Output { private String name; @SerializedName("text") - private List text; + private Object text; public List getText() { - return text; + List textList = new ArrayList<>(); + if (text instanceof String) { + textList.add((String) text); + } else { + textList = (List) text; + } + return textList; + } + + public boolean isError() { + if (name == null) { + return true; + } + return name.equals("stderr"); + } + + @Override + public ZeppelinOutputType getTypeOfZeppelin() { + return ZeppelinOutputType.TEXT; + } + + @Override + public TypeData toZeppelinResult() { + List text = verifyEndOfLine(getText()); + String result = Joiner.on("").join(text); + return new TypeData(getTypeOfZeppelin().toString(), result); } } diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/JupyterOutputType.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/JupyterOutputType.java new file mode 100644 index 0000000000..383ba11d5f --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/JupyterOutputType.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.types; + +/** + * Jupyter Output Types. + */ +public enum JupyterOutputType { + TEXT_PLAIN("text/plain"), + IMAGE_PNG("image/png"), + LATEX("text/latex"), + SVG_XML("image/svg+xml"), + TEXT_HTML("text/html"), + APPLICATION_JAVASCRIPT("application/javascript") + ; + + private final String type; + private JupyterOutputType(final String type) { + this.type = type; + } + + public ZeppelinOutputType getZeppelinType() { + return Convertor.ToZeppelin.getType(type); + } + + public static JupyterOutputType getByValue(String value) { + for (JupyterOutputType type : JupyterOutputType.values()) { + if (type.toString().equals(value)) { + return type; + } + } + return JupyterOutputType.TEXT_PLAIN; + } + + @Override + public String toString() { + return type; + } + + private enum Convertor { + ToZeppelin; + + public ZeppelinOutputType getType(String typeValue) { + JupyterOutputType type = JupyterOutputType.getByValue(typeValue); + ZeppelinOutputType outputType; + + if (JupyterOutputType.TEXT_PLAIN == type) { + outputType = ZeppelinOutputType.TEXT; + } else { + outputType = ZeppelinOutputType.HTML; + } + + return outputType; + } + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/ZeppelinOutputType.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/ZeppelinOutputType.java new file mode 100644 index 0000000000..cf1a5e0aa0 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/ZeppelinOutputType.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.types; + +/** + * Zeppelin Output Types. + */ +public enum ZeppelinOutputType { + TEXT("TEXT"), + HTML("HTML"), + TABLE("TABLE") + ; + + private final String type; + private ZeppelinOutputType(final String type) { + this.type = type; + } + + @Override + public String toString() { + return type; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java index f6a7ebfe6f..a93adae71e 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java @@ -19,6 +19,8 @@ package org.apache.zeppelin.jupyter.zformat; import com.google.gson.annotations.SerializedName; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; +import java.util.Map; /** * @@ -26,6 +28,9 @@ import java.util.Date; public class Paragraph { public static final String FINISHED = "FINISHED"; + @SerializedName("config") + private Map config; + @SerializedName("text") private String text; @@ -42,6 +47,22 @@ public class Paragraph { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd-HHmmss"); this.id = dateFormat.format(new Date()) + "_" + super.hashCode(); this.status = FINISHED; + initializeConfig(); + } + + private void initializeConfig() { + this.config = new HashMap<>(); + this.config.put("editorHide", false); + this.config.put("editorMode", "ace/mode/python"); + } + + public void setUpMarkdownConfig(boolean toActiveEditOnDblClickMode) { + Map editorSetting = new HashMap<>(); + editorSetting.put("language", "markdown"); + editorSetting.put("editOnDblClick", toActiveEditOnDblClickMode); + this.config.put("editorHide", toActiveEditOnDblClickMode); + this.config.put("editorSetting", editorSetting); + this.config.put("editorMode", "ace/mode/markdown"); } public String getText() { @@ -67,4 +88,8 @@ public class Paragraph { public String getStatus() { return status; } + + public Map getConfig() { + return config; + } } diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java index 7efa5c929d..a00cdd3231 100644 --- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java @@ -24,6 +24,7 @@ import java.util.List; */ public class Result { public static final String SUCCESS = "SUCCESS"; + public static final String ERROR = "ERROR"; @SerializedName("code") private String code; diff --git a/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java b/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java index be9a5ec966..a73571cec6 100644 --- a/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java +++ b/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java @@ -20,8 +20,14 @@ import static org.junit.Assert.assertTrue; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.List; +import java.util.Map; + +import com.google.gson.Gson; import org.apache.zeppelin.jupyter.JupyterUtil; import org.apache.zeppelin.jupyter.zformat.Note; +import org.apache.zeppelin.jupyter.zformat.Paragraph; +import org.apache.zeppelin.jupyter.zformat.TypeData; import org.junit.Test; /** @@ -30,7 +36,7 @@ import org.junit.Test; public class JupyterUtilTest { @Test - public void getNbFormat() { + public void getNbFormat() throws Exception { InputStream resource = getClass().getResourceAsStream("/basic.ipynb"); Nbformat nbformat = new JupyterUtil().getNbformat(new InputStreamReader(resource)); assertTrue(nbformat.getCells().get(0) instanceof CodeCell); @@ -40,9 +46,47 @@ public class JupyterUtilTest { } @Test - public void getNote() { + public void getNote() throws Exception { InputStream resource = getClass().getResourceAsStream("/examples.ipynb"); Note n = new JupyterUtil().getNote(new InputStreamReader(resource), "%python", "%md"); } + @Test + public void getNoteAndVerifyData() throws Exception { + String noteName = "Note converted from Jupyter"; + InputStream resource = getClass().getResourceAsStream("/basic.ipynb"); + Note n = new JupyterUtil().getNote(new InputStreamReader(resource), "%python", "%md"); + Gson gson = new Gson(); + System.out.println(gson.toJson(n)); + System.out.println(n.getParagraphs().size()); + assertTrue(n.getParagraphs().size() == 8); + assertTrue(noteName.equals(n.getName())); + + Paragraph firstParagraph = n.getParagraphs().get(0); + assertTrue(firstParagraph.getText().equals("%python\nimport numpy as np")); + assertTrue(firstParagraph.getStatus().equals("FINISHED")); + Map config = firstParagraph.getConfig(); + + assertTrue(((String) config.get("editorMode")).equals("ace/mode/python")); + assertTrue(((boolean) config.get("editorHide")) == false); + + Paragraph markdownParagraph = n.getParagraphs().get(6); + + assertTrue(markdownParagraph.getText().equals("%md\n" + + "
\n" + + "
\n" + + "
\n" + + "
This notebook was created using IBM Knowledge Anyhow Workbench. To learn more, visit us at https://knowledgeanyhow.org.
\n" + + "
\n" + + "
")); + assertTrue(markdownParagraph.getStatus().equals("FINISHED")); + + Map markdownConfig = markdownParagraph.getConfig(); + assertTrue(((String) markdownConfig.get("editorMode")).equals("ace/mode/markdown")); + assertTrue(((boolean) markdownConfig.get("editorHide")) == true); + assertTrue(markdownParagraph.getResults().getCode().equals("SUCCESS")); + List results = markdownParagraph.getResults().getMsg(); + assertTrue(results.get(0).getData().equals("\u003cdiv class\u003d\"alert\" style\u003d\"border: 1px solid #aaa; background: radial-gradient(ellipse at center, #ffffff 50%, #eee 100%);\"\u003e\n\u003cdiv class\u003d\"row\"\u003e\n \u003cdiv class\u003d\"col-sm-1\"\u003e\u003cimg src\u003d\"https://knowledgeanyhow.org/static/images/favicon_32x32.png\" style\u003d\"margin-top: -6px\"/\u003e\u003c/div\u003e\n \u003cdiv class\u003d\"col-sm-11\"\u003eThis notebook was created using \u003ca href\u003d\"https://knowledgeanyhow.org\"\u003eIBM Knowledge Anyhow Workbench\u003c/a\u003e. To learn more, visit us at \u003ca href\u003d\"https://knowledgeanyhow.org\"\u003ehttps://knowledgeanyhow.org\u003c/a\u003e.\u003c/div\u003e\n \u003c/div\u003e\n\u003c/div\u003e")); + assertTrue(results.get(0).getType().equals("HTML")); + } } diff --git a/zeppelin-jupyter/src/test/resources/basic.ipynb b/zeppelin-jupyter/src/test/resources/basic.ipynb index 472f868e63..6faf17e415 100644 --- a/zeppelin-jupyter/src/test/resources/basic.ipynb +++ b/zeppelin-jupyter/src/test/resources/basic.ipynb @@ -17,7 +17,7 @@ * under the License. */ - { +{ "cells": [ { "cell_type": "code", @@ -113,6 +113,38 @@ " print('4d}'.format(c) for c in coeffs[i]], ', ') + '}\"/>')" ] }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false, + "trusted": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n\n alt.atheism 0.86 0.95 0.90 137\n comp.graphics 0.89 0.84 0.86 178\n comp.os.ms-windows.misc 0.85 0.90 0.88 168\ncomp.sys.ibm.pc.hardware 0.85 0.78 0.81 180\n comp.sys.mac.hardware 0.95 0.83 0.89 174\n comp.windows.x 0.88 0.91 0.90 193\n misc.forsale 0.78 0.92 0.84 157\n rec.autos 0.93 0.89 0.91 188\n rec.motorcycles 0.96 0.95 0.95 185\n rec.sport.baseball 0.94 0.95 0.95 183\n rec.sport.hockey 0.92 0.98 0.95 180\n sci.crypt 0.95 0.97 0.96 181\n sci.electronics 0.93 0.80 0.86 180\n sci.med 0.95 0.94 0.95 159\n sci.space 0.91 0.98 0.95 185\n soc.religion.christian 0.88 0.92 0.90 176\n talk.politics.guns 0.93 0.98 0.95 176\n talk.politics.mideast 0.92 0.98 0.95 180\n talk.politics.misc 0.97 0.91 0.94 127\n talk.religion.misc 0.99 0.69 0.81 108\n\n avg / total 0.91 0.91 0.91 3395\n\n" + ] + } + ], + "source": [ + "# \n", + "# train the model and predict the test set\n", + "y_pred = clf.fit(X_train, y_train).predict(X_test)\n", + "\n", + "# standard information retrieval metrics\n", + "print metrics.classification_report(y_test, y_pred, target_names=labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n
\n
\n
This notebook was created using IBM Knowledge Anyhow Workbench. To learn more, visit us at https://knowledgeanyhow.org.
\n
\n
" + ] + }, { "cell_type": "code", "execution_count": null, From 780f0ebb43025c2652e5bbbe1b75fb598035f5b6 Mon Sep 17 00:00:00 2001 From: krishna-pandey Date: Thu, 20 Jul 2017 00:26:01 +0530 Subject: [PATCH 02/82] [ZEPPELIN-2775] Strict-Transport-Security and X-XSS-Protection Headers ### What is this PR for? The HTTP Strict-Transport-Security response header (often abbreviated as HSTS) is a security feature that lets a web site tell browsers that it should only be communicated with using HTTPS, instead of using HTTP. Note: The Strict-Transport-Security header is ignored by the browser when your site is accessed using HTTP; this is because an attacker may intercept HTTP connections and inject the header or remove it. When your site is accessed over HTTPS with no certificate errors, the browser knows your site is HTTPS capable and will honor the Strict-Transport-Security header. The HTTP X-XSS-Protection response header is a feature of Internet Explorer, Chrome and Safari that stops pages from loading when they detect reflected cross-site scripting (XSS) attacks. ### What type of PR is it? [Bug Fix | Improvement ] ### What is the Jira issue? * [ZEPPELIN-2775](https://issues.apache.org/jira/browse/ZEPPELIN-2775) ### How should this be tested? Make a curl call to Zeppelin? Go to Chrome Browser and select "More Tools" -> "Developer Tools" from the right-side menu. Under Network Section, select any request and check for "Response Headers". You should see below headers along with existing ones. > strict-transport-security:max-age=631138519 > x-xss-protection:1; mode=block screen shot 2017-07-14 at 8 19 14 pm ### Questions: * Does this needs documentation? Author: krishna-pandey Closes #2492 from krishna-pandey/ZEPPELIN-2775 and squashes the following commits: 7d9978e49 [krishna-pandey] Modified Documentation as per review. 6733289ed [krishna-pandey] Adding documentation for HTTP Security Headers 754d2d71e [krishna-pandey] Supplying String instead of Int (required for Response Header) 468231cc6 [krishna-pandey] Added configurable Strict-Transport-Security and X-XSS-Protection Headers --- conf/zeppelin-site.xml.template | 14 +++ .../themes/zeppelin/_navigation.html | 1 + docs/index.md | 1 + docs/setup/security/http_security_headers.md | 110 ++++++++++++++++++ .../apache/zeppelin/server/CorsFilter.java | 7 +- .../zeppelin/conf/ZeppelinConfiguration.java | 12 +- 6 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 docs/setup/security/http_security_headers.md diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index cbae4e5900..adf58102ca 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -396,4 +396,18 @@ --> + + diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 906d6e8806..ecdccbd7ff 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -94,6 +94,7 @@
  • Shiro Authentication
  • Notebook Authorization
  • Data Source Authorization
  • +
  • HTTP Security Headers
  • Notebook Storage
  • Git Storage
  • diff --git a/docs/index.md b/docs/index.md index 95104a76df..102af4cdbe 100644 --- a/docs/index.md +++ b/docs/index.md @@ -97,6 +97,7 @@ limitations under the License. * [Shiro Authentication](./setup/security/shiro_authentication.html) * [Notebook Authorization](./setup/security/notebook_authorization.html) * [Data Source Authorization](./setup/security/datasource_authorization.html) + * [HTTP Security Headers](./setup/security/http_security_headers.html) * Notebook Storage: a guide about saving notebooks to external storage * [Git Storage](./setup/storage/storage.html#notebook-storage-in-local-git-repository) * [S3 Storage](./setup/storage/storage.html#notebook-storage-in-s3) diff --git a/docs/setup/security/http_security_headers.md b/docs/setup/security/http_security_headers.md new file mode 100644 index 0000000000..1c55d18e18 --- /dev/null +++ b/docs/setup/security/http_security_headers.md @@ -0,0 +1,110 @@ +--- +layout: page +title: "Setting up HTTP Response Headers" +description: "There are multiple HTTP Security Headers which can be configured in Apache Zeppelin. This page describes how to enable them by providing appropriate value in Zeppelin configuration file." +group: setup/security +--- + +{% include JB/setup %} + +# Setting up HTTP Response Headers for Zeppelin + +
    + +Apache Zeppelin can be configured to include HTTP Headers which aids in preventing Cross Site Scripting (XSS), Cross-Frame Scripting (XFS) and also enforces HTTP Strict Transport Security. Apache Zeppelin also has configuration available to set the Application Server Version to desired value. + +## Setting up HTTP Strict Transport Security (HSTS) Response Header + +Enabling HSTS Response Header prevents Man-in-the-middle attacks by automatically redirecting HTTP requests to HTTPS when Zeppelin Server is running on SSL. Read on how to configure SSL for Zeppelin [here] (../operation/configuration.html). Even if web page contains any resource which gets served over HTTP or any HTTP links, it will automatically be redirected to HTTPS for the target domain. +It also prevents MITM attack by not allowing User to override the invalid certificate message, when Attacker presents invalid SSL certificate to the User. + +The following property needs to be updated in the zeppelin-site.xml in order to enable HSTS. You can choose appropriate value for "max-age". + +``` + + zeppelin.server.strict.transport + max-age=631138519 + The HTTP Strict-Transport-Security response header is a security feature that lets a web site tell browsers that it should only be communicated with using HTTPS, instead of using HTTP. Enable this when Zeppelin is running on HTTPS. Value is in Seconds, the default value is equivalent to 20 years. + +``` + + +Possible values are: + +* max-age=\ +* max-age=\; includeSubDomains +* max-age=\; preload + +Read more about HSTS [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security). + +## Setting up X-XSS-PROTECTION Header + +The HTTP X-XSS-Protection response header is a feature of Internet Explorer, Chrome and Safari Web browsers that initiates configured action when they detect reflected cross-site scripting (XSS) attacks. + +The following property needs to be updated in the zeppelin-site.xml in order to set X-XSS-PROTECTION header. + +``` + + zeppelin.server.xxss.protection + 1; mode=block + The HTTP X-XSS-Protection response header is a feature of Internet Explorer, Chrome and Safari that stops pages from loading when they detect reflected cross-site scripting (XSS) attacks. When value is set to 1 and a cross-site scripting attack is detected, the browser will sanitize the page (remove the unsafe parts). + +``` + + +You can choose appropriate value from below. + +* 0 (Disables XSS filtering) +* 1 (Enables XSS filtering. If a cross-site scripting attack is detected, the browser will sanitize the page.) +* 1; mode=block (Enables XSS filtering. The browser will prevent rendering of the page if an attack is detected.) + +Read more about HTTP X-XSS-Protection response header [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection). + +## Setting up X-Frame-Options Header + +The X-Frame-Options HTTP response header can indicate browser to avoid clickjacking attacks, by ensuring that their content is not embedded into other sites in a ``,`