diff --git a/.travis.yml b/.travis.yml index 61305091c2..5a728f72e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,8 +46,12 @@ matrix: env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS="" # Test core modules + # + # Several tests were excluded from this configuration due to the following issues: + # HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470 + # After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property - jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtest='!ZeppelinSparkClusterTest,!org.apache.zeppelin.spark.*' -DfailIfNoTests=false" + env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk7" @@ -55,7 +59,7 @@ matrix: # Test interpreter modules - jdk: "oraclejdk7" - env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" + env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" # Test spark module for 2.1.0 with scala 2.11, livy - jdk: "oraclejdk7" @@ -89,7 +93,7 @@ before_install: - changedfiles=$(git diff --name-only $TRAVIS_COMMIT_RANGE 2>/dev/null) || changedfiles="" - echo $changedfiles - hasbowerchanged=$(echo $changedfiles | grep -c "bower.json" || true); - - gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog="" + - gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog="" - clearcache=$(echo $gitlog | grep -c -E "clear bower|bower clear" || true) - if [ "$hasbowerchanged" -gt 0 ] || [ "$clearcache" -gt 0 ]; then echo "Clearing bower_components cache"; rm -r zeppelin-web/bower_components; npm cache clear; else echo "Using cached bower_components."; fi - echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc diff --git a/conf/shiro.ini.template b/conf/shiro.ini.template index 0a3187a645..06ad9712a5 100644 --- a/conf/shiro.ini.template +++ b/conf/shiro.ini.template @@ -18,7 +18,8 @@ [users] # List of users with their password allowed to access Zeppelin. # To use a different strategy (LDAP / Database / ...) check the shiro doc at http://shiro.apache.org/configuration.html#Configuration-INISections -admin = password1, admin +# To enable admin user, uncomment the following line and set an appropriate password. +#admin = password1, admin user1 = password2, role1, role2 user2 = password3, role3 user3 = password4, role2 @@ -73,14 +74,22 @@ role3 = * admin = * [urls] -# This section is used for url-based security. -# You can secure interpreter, configuration and credential information by urls. Comment or uncomment the below urls that you want to hide. +# This section is used for url-based security. For details see the shiro.ini documentation. +# +# You can secure interpreter, configuration and credential information by urls. +# Comment or uncomment the below urls that you want to hide: # anon means the access is anonymous. -# authc means Form based Auth Security -# To enfore security, comment the line below and uncomment the next one +# authc means form based auth Security. +# +# IMPORTANT: Order matters: URL path expressions are evaluated against an incoming request +# in the order they are defined and the FIRST MATCH WINS. +# +# To allow anonymous access to all but the stated urls, +# uncomment the line second last line (/** = anon) and comment the last line (/** = authc) +# /api/version = anon -#/api/interpreter/** = authc, roles[admin] -#/api/configurations/** = authc, roles[admin] -#/api/credential/** = authc, roles[admin] +/api/interpreter/** = authc, roles[admin] +/api/configurations/** = authc, roles[admin] +/api/credential/** = authc, roles[admin] #/** = anon /** = authc diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template index cd1541e72c..e40429af9e 100644 --- a/conf/zeppelin-env.cmd.template +++ b/conf/zeppelin-env.cmd.template @@ -39,7 +39,9 @@ REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zep REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading REM set ZEPPELIN_INTERPRETER_DEP_MVNREPO REM Maven principal repository for interpreter's additional dependency loading -REM set ZEPPELIN_HELIUM_NPM_REGISTRY REM Remote Npm registry for Helium dependency loader +REM set ZEPPELIN_HELIUM_NODE_INSTALLER_URL REM Remote Node installer url for Helium dependency loader +REM set ZEPPELIN_HELIUM_NPM_INSTALLER_URL REM Remote Npm installer url for Helium dependency loader +REM set ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL REM Remote Yarn package installer url for Helium dependency loader REM set ZEPPELIN_NOTEBOOK_STORAGE REM Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). REM set ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC REM If there are multiple notebook storages, should we treat the first one as the only source of truth? diff --git a/conf/zeppelin-env.sh.template b/conf/zeppelin-env.sh.template index 754c8ef349..ce55346fcd 100644 --- a/conf/zeppelin-env.sh.template +++ b/conf/zeppelin-env.sh.template @@ -44,7 +44,9 @@ # export ZEPPELIN_NICENESS # The scheduling priority for daemons. Defaults to 0. # export ZEPPELIN_INTERPRETER_LOCALREPO # Local repository for interpreter's additional dependency loading # export ZEPPELIN_INTERPRETER_DEP_MVNREPO # Remote principal repository for interpreter's additional dependency loading -# export ZEPPELIN_HELIUM_NPM_REGISTRY # Remote Npm registry for Helium dependency loader +# export ZEPPELIN_HELIUM_NODE_INSTALLER_URL # Remote Node installer url for Helium dependency loader +# export ZEPPELIN_HELIUM_NPM_INSTALLER_URL # Remote Npm installer url for Helium dependency loader +# export ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL # Remote Yarn package installer url for Helium dependency loader # export ZEPPELIN_NOTEBOOK_STORAGE # Refers to pluggable notebook storage class, can have two classes simultaneously with a sync between them (e.g. local and remote). # export ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC # If there are multiple notebook storages, should we treat the first one as the only source of truth? # export ZEPPELIN_NOTEBOOK_PUBLIC # Make notebook public by default when created, private otherwise diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 2a03cd9d46..584434b9d5 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -252,9 +252,21 @@ - zeppelin.helium.npm.registry + zeppelin.helium.node.installer.url + https://nodejs.org/dist/ + Remote Node installer url for Helium dependency loader + + + + zeppelin.helium.npm.installer.url http://registry.npmjs.org/ - Remote Npm registry for Helium dependency loader + Remote Npm installer url for Helium dependency loader + + + + zeppelin.helium.yarnpkg.installer.url + https://github.com/yarnpkg/yarn/releases/download/ + Remote Yarn package installer url for Helium dependency loader diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 4e49a1acb3..dd710d07b7 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -88,6 +88,7 @@
  • Text
  • Html
  • Table
  • +
  • Network
  • Angular API
  • Angular (backend API)
  • @@ -105,6 +106,7 @@
  • MongoDB Storage
  • REST API
  • +
  • Zeppelin Server API
  • Interpreter API
  • Notebook API
  • Notebook Repository API
  • @@ -116,6 +118,7 @@
  • Shiro Authentication
  • Notebook Authorization
  • Data Source Authorization
  • +
  • Helium Authorization
  • Helium Framework (Experimental)
  • Writing Zeppelin Application
  • diff --git a/docs/assets/themes/zeppelin/img/screenshots/display_complex_network.png b/docs/assets/themes/zeppelin/img/screenshots/display_complex_network.png new file mode 100644 index 0000000000..c788c26004 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/screenshots/display_complex_network.png differ diff --git a/docs/assets/themes/zeppelin/img/screenshots/display_network.png b/docs/assets/themes/zeppelin/img/screenshots/display_network.png new file mode 100644 index 0000000000..516c85332f Binary files /dev/null and b/docs/assets/themes/zeppelin/img/screenshots/display_network.png differ diff --git a/docs/assets/themes/zeppelin/img/screenshots/display_network_flatten.png b/docs/assets/themes/zeppelin/img/screenshots/display_network_flatten.png new file mode 100644 index 0000000000..743e6666c6 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/screenshots/display_network_flatten.png differ diff --git a/docs/assets/themes/zeppelin/img/screenshots/display_simple_network.png b/docs/assets/themes/zeppelin/img/screenshots/display_simple_network.png new file mode 100644 index 0000000000..86a08fe02a Binary files /dev/null and b/docs/assets/themes/zeppelin/img/screenshots/display_simple_network.png differ diff --git a/docs/displaysystem/basicdisplaysystem.md b/docs/displaysystem/basicdisplaysystem.md index 7c4243208e..15cefefbb5 100644 --- a/docs/displaysystem/basicdisplaysystem.md +++ b/docs/displaysystem/basicdisplaysystem.md @@ -61,3 +61,104 @@ If table contents start with `%html`, it is interpreted as an HTML. > **Note :** Display system is backend independent. + +## Network + +With the `%network` directive, Zeppelin treats your output as a graph. Zeppelin can leverage the Property Graph Model. + +### What is the Labelled Property Graph Model? + +A [Property Graph](https://github.com/tinkerpop/gremlin/wiki/Defining-a-Property-Graph) is a graph that has these elements: + +* a set of vertices + * each vertex has a unique identifier. + * each vertex has a set of outgoing edges. + * each vertex has a set of incoming edges. + * each vertex has a collection of properties defined by a map from key to value +* a set of edges + * each edge has a unique identifier. + * each edge has an outgoing tail vertex. + * each edge has an incoming head vertex. + * each edge has a label that denotes the type of relationship between its two vertices. + * each edge has a collection of properties defined by a map from key to value. + + + +A [Labelled Property Graph](https://neo4j.com/developer/graph-database/#property-graph) is a Property Graph where the nodes can be tagged with **labels** representing their different roles in the graph model + + + +### What are the APIs? + +The new NETWORK visualization is based on json with the following params: + +* "nodes" (mandatory): list of nodes of the graph every node can have the following params: + * "id" (mandatory): the id of the node (must be unique); + * "label": the main Label of the node; + * "labels": the list of the labels of the node; + * "data": the data attached to the node; +* "edges": list of the edges of the graph; + * "id" (mandatory): the id of the edge (must be unique); + * "source" (mandatory): the id of source node of the edge; + * "target" (mandatory): the id of target node of the edge; + * "label": the main type of the edge; + * "data": the data attached to the edge; +* "labels": a map (K, V) where K is the node label and V is the color of the node; +* "directed": (true/false, default false) wich tells if is directed graph or not; +* "types": a *distinct* list of the edge types of the graph + +If you click on a node or edge on the bottom of the paragraph you find a list of entity properties + + + +This kind of graph can be easily *flatten* in order to support other visualization formats provided by Zeppelin. + + + +### How to use it? + +An example of a simple graph + +``` +%spark +print(s""" +%network { + "nodes": [ + {"id": 1}, + {"id": 2}, + {"id": 3} + ], + "edges": [ + {"source": 1, "target": 2, "id" : 1}, + {"source": 2, "target": 3, "id" : 2}, + {"source": 1, "target": 2, "id" : 3}, + {"source": 1, "target": 2, "id" : 4}, + {"source": 2, "target": 1, "id" : 5}, + {"source": 2, "target": 1, "id" : 6} + ] +} +""") +``` + +that will look like: + + + +A little more complex graph: + +``` +%spark +print(s""" +%network { + "nodes": [{"id": 1, "label": "User", "data": {"fullName":"Andrea Santurbano"}},{"id": 2, "label": "User", "data": {"fullName":"Lee Moon Soo"}},{"id": 3, "label": "Project", "data": {"name":"Zeppelin"}}], + "edges": [{"source": 2, "target": 1, "id" : 1, "label": "HELPS"},{"source": 2, "target": 3, "id" : 2, "label": "CREATE"},{"source": 1, "target": 3, "id" : 3, "label": "CONTRIBUTE_TO", "data": {"oldPR": "https://github.com/apache/zeppelin/pull/1582"}}], + "labels": {"User": "#8BC34A", "Project": "#3071A9"}, + "directed": true, + "types": ["HELPS", "CREATE", "CONTRIBUTE_TO"] +} +""") +``` + +that will look like: + + \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 043538d217..bb2222b932 100644 --- a/docs/index.md +++ b/docs/index.md @@ -161,6 +161,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor * [Azure Storage](./storage/storage.html#notebook-storage-in-azure) * [ZeppelinHub Storage](./storage/storage.html#storage-in-zeppelinhub) * REST API: available REST API list in Apache Zeppelin + * [Zeppelin server API](./rest-api/rest-zeppelin-server.html) * [Interpreter API](./rest-api/rest-interpreter.html) * [Notebook API](./rest-api/rest-notebook.html) * [Notebook Repository API](./rest-api/rest-notebookRepo.html) @@ -172,6 +173,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor * [Shiro Authentication](./security/shiroauthentication.html) * [Notebook Authorization](./security/notebook_authorization.html) * [Data Source Authorization](./security/datasource_authorization.html) + * [Helium Authorization](./security/helium_authorization.html) * Helium Framework (Experimental) * [Writing Zeppelin Application](./development/writingzeppelinapplication.html) * [Writing Zeppelin Spell](./development/writingzeppelinspell.html) diff --git a/docs/install/configuration.md b/docs/install/configuration.md index a35eb91594..ca55397828 100644 --- a/docs/install/configuration.md +++ b/docs/install/configuration.md @@ -282,10 +282,22 @@ If both are defined, then the **environment variables** will take priority. Local repository for dependency loader.
    ex)visualiztion modules of npm. -
    ZEPPELIN_HELIUM_NPM_REGISTRY
    -
    zeppelin.helium.npm.registry
    +
    ZEPPELIN_HELIUM_NODE_INSTALLER_URL
    +
    zeppelin.helium.node.installer.url
    + https://nodejs.org/dist/ + Remote Node installer url for Helium dependency loader + + +
    ZEPPELIN_HELIUM_NPM_INSTALLER_URL
    +
    zeppelin.helium.npm.installer.url
    http://registry.npmjs.org/ - Remote Npm registry for Helium dependency loader + Remote Npm installer url for Helium dependency loader + + +
    ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL
    +
    zeppelin.helium.yarnpkg.installer.url
    + https://github.com/yarnpkg/yarn/releases/download/ + Remote Yarn package installer url for Helium dependency loader
    ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE
    diff --git a/docs/interpreter/jdbc.md b/docs/interpreter/jdbc.md index b7ac45ae44..0fe9d68ecf 100644 --- a/docs/interpreter/jdbc.md +++ b/docs/interpreter/jdbc.md @@ -128,6 +128,16 @@ The JDBC interpreter properties are defined by default like below. Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info) + + default.completer.ttlInSeconds + 120 + Time to live sql completer in seconds (-1 to update everytime, 0 to disable update) + + + default.splitQueries + false + Each query is executed apart and returns the result + If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values. diff --git a/docs/rest-api/rest-configuration.md b/docs/rest-api/rest-configuration.md index 47c65f11d1..6bfa62e459 100644 --- a/docs/rest-api/rest-configuration.md +++ b/docs/rest-api/rest-configuration.md @@ -29,7 +29,6 @@ All REST APIs are available starting with the following endpoint `http://[zeppel Note that Apache Zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc). If you work with Apache Zeppelin and find a need for an additional REST API, please [file an issue or send us an email](http://zeppelin.apache.org/community.html). -nd a need for an additional REST API, please [file an issue or send us mail](../../community.html). ## Configuration REST API list diff --git a/docs/rest-api/rest-zeppelin-server.md b/docs/rest-api/rest-zeppelin-server.md new file mode 100644 index 0000000000..750d5dc79a --- /dev/null +++ b/docs/rest-api/rest-zeppelin-server.md @@ -0,0 +1,78 @@ +--- +layout: page +title: "Apache Zeppelin Server REST API" +description: "This page contains Apache Zeppelin Server REST API information." +group: rest-api +--- + +{% include JB/setup %} + +# Apache Zeppelin Server REST API + +
    + +## Overview +Apache Zeppelin provides several REST APIs for interaction and remote activation of zeppelin functionality. +All REST APIs are available starting with the following endpoint `http://[zeppelin-server]:[zeppelin-port]/api`. +Note that Apache Zeppelin REST APIs receive or return JSON objects, it is recommended for you to install some JSON viewers such as [JSONView](https://chrome.google.com/webstore/detail/jsonview/chklaanhfefbnpoihckbnefhakgolnmc). + +If you work with Apache Zeppelin and find a need for an additional REST API, please [file an issue or send us an email](http://zeppelin.apache.org/community.html). + + +## Zeppelin Server REST API list + +### Change the log level of Zeppelin Server + + + + + + + + + + + + + + + + + + + + + + + + + + +
    DescriptionThis ```PUT``` method is used to update the root logger's log level of the server.
    URL```http://[zeppelin-server]:[zeppelin-port]/api/log/level/```
    Success code200
    Fail code 406
    sample JSON response + +
    +{
    +  "status": "OK"
    +}     
    +        
    +
    sample error JSON response + +
    +{
    +  "status":"NOT_ACCEPTABLE",
    +  "message":"Please check LOG level specified. Valid values: DEBUG, ERROR, FATAL, INFO, TRACE, WARN"
    +}     
    +        
    +
    diff --git a/docs/security/helium_authorization.md b/docs/security/helium_authorization.md new file mode 100644 index 0000000000..fdfb435dfc --- /dev/null +++ b/docs/security/helium_authorization.md @@ -0,0 +1,28 @@ +--- +layout: page +title: "Helium Authorization in Apache Zeppelin" +description: "Apache Zeppelin supports Helium plugins which fetch required installer packages from remote registry/repositories" +group: security +--- + +{% include JB/setup %} + +# Helium Authorization in Apache Zeppelin + +
    + +## How to configure proxies? + +Set **http_proxy** and **https_proxy** env variables to allow connection to npm registry behind a corporate firewall. \ No newline at end of file diff --git a/docs/security/shiroauthentication.md b/docs/security/shiroauthentication.md index 7fd29ee94f..7fce0291c4 100644 --- a/docs/security/shiroauthentication.md +++ b/docs/security/shiroauthentication.md @@ -128,6 +128,10 @@ Change the following values in the Shiro.ini file, and uncomment the line: ### LDAP +Two options exist for configuring an LDAP Realm. The simpler to use is the LdapGroupRealm. How ever it has limited +flexibility with mapping of ldap groups to users and for authorization for user groups. A sample configuration file for +this realm is given below. + ``` ldapRealm = org.apache.zeppelin.realm.LdapGroupRealm # search base for ldap groups (only relevant for LdapGroupRealm): @@ -137,6 +141,46 @@ ldapRealm.userDnTemplate = uid={0},ou=Users,dc=COMPANY,dc=COM ldapRealm.contextFactory.authenticationMechanism = simple ``` +The other more flexible option is to use the LdapRealm. It allows for mapping of ldapgroups to roles and also allows for + role/group based authentication into the zeppelin server. Sample configuration for this realm is given below. + ``` +[main] +ldapRealm=org.apache.zeppelin.realm.LdapRealm + +ldapRealm.contextFactory.authenticationMechanism=simple +ldapRealm.contextFactory.url=ldap://localhost:33389 +ldapRealm.userDnTemplate=uid={0},ou=people,dc=hadoop,dc=apache,dc=org +# Ability to set ldap paging Size if needed default is 100 +ldapRealm.pagingSize = 200 +ldapRealm.authorizationEnabled=true +ldapRealm.contextFactory.systemAuthenticationMechanism=simple +ldapRealm.searchBase=dc=hadoop,dc=apache,dc=org +ldapRealm.userSearchBase = dc=hadoop,dc=apache,dc=org +ldapRealm.groupSearchBase = ou=groups,dc=hadoop,dc=apache,dc=org +ldapRealm.groupObjectClass=groupofnames +# Allow userSearchAttribute to be customized +ldapRealm.userSearchAttributeName = sAMAccountName +ldapRealm.memberAttribute=member +# force usernames returned from ldap to lowercase useful for AD +ldapRealm.userLowerCase = true +# ability set searchScopes subtree (default), one, base +ldapRealm.userSearchScope = subtree; +ldapRealm.groupSearchScope = subtree; +ldapRealm.memberAttributeValueTemplate=cn={0},ou=people,dc=hadoop,dc=apache,dc=org +ldapRealm.contextFactory.systemUsername=uid=guest,ou=people,dc=hadoop,dc=apache,dc=org +ldapRealm.contextFactory.systemPassword=S{ALIAS=ldcSystemPassword} +# enable support for nested groups using the LDAP_MATCHING_RULE_IN_CHAIN operator +ldapRealm.groupSearchEnableMatchingRuleInChain = true +# optional mapping from physical groups to logical application roles +ldapRealm.rolesByGroup = LDN_USERS: user_role, NYK_USERS: user_role, HKG_USERS: user_role, GLOBAL_ADMIN: admin_role +# optional list of roles that are allowed to authenticate. Incase not present all groups are allowed to authenticate (login). +# This changes nothing for url specific permissions that will continue to work as specified in [urls]. +ldapRealm.allowedRolesForAuthentication = admin_role,user_role +ldapRealm.permissionsByRole= user_role = *:ToDoItemsJdo:*:*, *:ToDoItem:*:*; admin_role = * +securityManager.sessionManager = $sessionManager +securityManager.realms = $ldapRealm + ``` + ### PAM [PAM](https://en.wikipedia.org/wiki/Pluggable_authentication_module) authentication support allows the reuse of existing authentication moduls on the host where Zeppelin is running. On a typical system modules are configured per service for example sshd, passwd, etc. under `/etc/pam.d/`. You can diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index b75d8b85fd..72d7981764 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -23,11 +23,15 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Properties; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.commons.dbcp2.ConnectionFactory; import org.apache.commons.dbcp2.DriverManagerConnectionFactory; @@ -101,6 +105,9 @@ public class JDBCInterpreter extends Interpreter { static final String PASSWORD_KEY = "password"; static final String PRECODE_KEY = "precode"; static final String COMPLETER_SCHEMA_FILTERS_KEY = "completer.schemaFilters"; + static final String COMPLETER_TTL_KEY = "completer.ttlInSeconds"; + static final String DEFAULT_COMPLETER_TTL = "120"; + static final String SPLIT_QURIES_KEY = "splitQueries"; static final String JDBC_JCEKS_FILE = "jceks.file"; static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey"; static final String PRECODE_KEY_TEMPLATE = "%s.precode"; @@ -128,6 +135,7 @@ public class JDBCInterpreter extends Interpreter { private final HashMap basePropretiesMap; private final HashMap jdbcUserConfigurationsMap; + private final HashMap sqlCompletersMap; private int maxLineResults; @@ -135,6 +143,7 @@ public class JDBCInterpreter extends Interpreter { super(property); jdbcUserConfigurationsMap = new HashMap<>(); basePropretiesMap = new HashMap<>(); + sqlCompletersMap = new HashMap<>(); maxLineResults = MAX_LINE_DEFAULT; } @@ -188,11 +197,43 @@ public class JDBCInterpreter extends Interpreter { } } - private SqlCompleter createSqlCompleter(Connection jdbcConnection, String propertyKey) { + private SqlCompleter createOrUpdateSqlCompleter(SqlCompleter sqlCompleter, + final Connection connection, String propertyKey, final String buf, final int cursor) { String schemaFiltersKey = String.format("%s.%s", propertyKey, COMPLETER_SCHEMA_FILTERS_KEY); - String filters = getProperty(schemaFiltersKey); - SqlCompleter completer = new SqlCompleter(); - completer.initFromConnection(jdbcConnection, filters); + String sqlCompleterTtlKey = String.format("%s.%s", propertyKey, COMPLETER_TTL_KEY); + final String schemaFiltersString = getProperty(schemaFiltersKey); + int ttlInSeconds = Integer.valueOf( + StringUtils.defaultIfEmpty(getProperty(sqlCompleterTtlKey), DEFAULT_COMPLETER_TTL) + ); + final SqlCompleter completer; + if (sqlCompleter == null) { + completer = new SqlCompleter(ttlInSeconds); + } else { + completer = sqlCompleter; + } + ExecutorService executorService = Executors.newFixedThreadPool(1); + executorService.execute(new Runnable() { + @Override + public void run() { + completer.createOrUpdateFromConnection(connection, schemaFiltersString, buf, cursor); + } + }); + + executorService.shutdown(); + + try { + // protection to release connection + executorService.awaitTermination(3, TimeUnit.SECONDS); + } catch (InterruptedException e) { + logger.warn("Completion timeout", e); + if (connection != null) { + try { + connection.close(); + } catch (SQLException e1) { + logger.warn("Error close connection", e1); + } + } + } return completer; } @@ -499,7 +540,6 @@ public class JDBCInterpreter extends Interpreter { StringBuilder query = new StringBuilder(); char character; - Boolean antiSlash = false; Boolean multiLineComment = false; Boolean singleLineComment = false; Boolean quoteString = false; @@ -522,14 +562,8 @@ public class JDBCInterpreter extends Interpreter { continue; } - if (character == '\\') { - antiSlash = true; - } - if (character == '\'') { - if (antiSlash) { - antiSlash = false; - } else if (quoteString) { + if (quoteString) { quoteString = false; } else if (!doubleQuoteString) { quoteString = true; @@ -537,9 +571,7 @@ public class JDBCInterpreter extends Interpreter { } if (character == '"') { - if (antiSlash) { - antiSlash = false; - } else if (doubleQuoteString) { + if (doubleQuoteString && item > 0) { doubleQuoteString = false; } else if (!quoteString) { doubleQuoteString = true; @@ -559,7 +591,7 @@ public class JDBCInterpreter extends Interpreter { } } - if (character == ';' && !antiSlash && !quoteString && !doubleQuoteString) { + if (character == ';' && !quoteString && !doubleQuoteString) { queries.add(StringUtils.trim(query.toString())); query = new StringBuilder(); } else if (item == sql.length() - 1) { @@ -596,17 +628,29 @@ public class JDBCInterpreter extends Interpreter { String paragraphId = interpreterContext.getParagraphId(); String user = interpreterContext.getAuthenticationInfo().getUser(); - InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); + boolean splitQuery = false; + String splitQueryProperty = getProperty(String.format("%s.%s", propertyKey, SPLIT_QURIES_KEY)); + if (StringUtils.isNotBlank(splitQueryProperty) && splitQueryProperty.equalsIgnoreCase("true")) { + splitQuery = true; + } + InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); try { connection = getConnection(propertyKey, interpreterContext); if (connection == null) { return new InterpreterResult(Code.ERROR, "Prefix not found."); } - ArrayList multipleSqlArray = splitSqlQueries(sql); - for (int i = 0; i < multipleSqlArray.size(); i++) { - String sqlToExecute = multipleSqlArray.get(i); + + List sqlArray; + if (splitQuery) { + sqlArray = splitSqlQueries(sql); + } else { + sqlArray = Arrays.asList(sql); + } + + for (int i = 0; i < sqlArray.size(); i++) { + String sqlToExecute = sqlArray.get(i); statement = connection.createStatement(); if (statement == null) { return new InterpreterResult(Code.ERROR, "Prefix not found."); @@ -787,6 +831,10 @@ public class JDBCInterpreter extends Interpreter { InterpreterContext interpreterContext) { List candidates = new ArrayList<>(); String propertyKey = getPropertyKey(buf); + String sqlCompleterKey = + String.format("%s.%s", interpreterContext.getAuthenticationInfo().getUser(), propertyKey); + SqlCompleter sqlCompleter = sqlCompletersMap.get(sqlCompleterKey); + Connection connection = null; try { if (interpreterContext != null) { @@ -796,11 +844,9 @@ public class JDBCInterpreter extends Interpreter { logger.warn("SQLCompleter will created without use connection"); } - SqlCompleter sqlCompleter = createSqlCompleter(connection, propertyKey); - - if (sqlCompleter != null) { - sqlCompleter.complete(buf, cursor - 1, candidates); - } + sqlCompleter = createOrUpdateSqlCompleter(sqlCompleter, connection, propertyKey, buf, cursor); + sqlCompletersMap.put(sqlCompleterKey, sqlCompleter); + sqlCompleter.complete(buf, cursor, candidates); return candidates; } diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java index 704ec597c4..46cc4bd0c2 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java @@ -24,6 +24,7 @@ import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; +import org.apache.zeppelin.completer.CachedCompleter; import org.apache.zeppelin.completer.CompletionType; import org.apache.zeppelin.completer.StringsCompleter; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -33,8 +34,6 @@ import org.slf4j.LoggerFactory; import jline.console.completer.ArgumentCompleter.ArgumentList; import jline.console.completer.ArgumentCompleter.WhitespaceArgumentDelimiter; -import static org.apache.commons.lang.StringUtils.isBlank; - /** * SQL auto complete functionality for the JdbcInterpreter. */ @@ -42,6 +41,7 @@ public class SqlCompleter { private static Logger logger = LoggerFactory.getLogger(SqlCompleter.class); + /** * Delimiter that can split SQL statement in keyword list */ @@ -59,23 +59,30 @@ public class SqlCompleter { /** * Schema completer */ - private StringsCompleter schemasCompleter = new StringsCompleter(); + private CachedCompleter schemasCompleter; /** * Contain different completer with table list for every schema name */ - private Map tablesCompleters = new HashMap<>(); + private Map tablesCompleters = new HashMap<>(); /** * Contains different completer with column list for every table name * Table names store as schema_name.table_name */ - private Map columnsCompleters = new HashMap<>(); + private Map columnsCompleters = new HashMap<>(); /** * Completer for sql keywords */ - private StringsCompleter keywordCompleter = new StringsCompleter(); + private CachedCompleter keywordCompleter; + + private int ttlInSeconds; + + + public SqlCompleter(int ttlInSeconds) { + this.ttlInSeconds = ttlInSeconds; + } public int complete(String buffer, int cursor, List candidates) { @@ -95,25 +102,9 @@ public class SqlCompleter { argumentPosition = argumentList.getArgumentPosition(); } - boolean isColumnAllowed = true; - if (buffer.length() > 0) { - String beforeCursorBuffer = buffer.substring(0, - Math.min(cursor, buffer.length())).toUpperCase(); - // check what sql is and where cursor is to allow column completion or not - if (beforeCursorBuffer.contains("SELECT ") && beforeCursorBuffer.contains(" FROM ") - && !beforeCursorBuffer.contains(" WHERE ")) - isColumnAllowed = false; - } - int complete = completeName(cursorArgument, argumentPosition, candidates, - findAliasesInSQL(argumentList.getArguments()), isColumnAllowed); + findAliasesInSQL(argumentList.getArguments())); - if (candidates.size() == 1) { - InterpreterCompletion interpreterCompletion = candidates.get(0); - interpreterCompletion.setName(interpreterCompletion.getName() + " "); - interpreterCompletion.setValue(interpreterCompletion.getValue() + " "); - candidates.set(0, interpreterCompletion); - } logger.debug("complete:" + complete + ", size:" + candidates.size()); return complete; } @@ -132,6 +123,7 @@ public class SqlCompleter { Set res = new HashSet<>(); try { ResultSet schemas = meta.getSchemas(); + try { while (schemas.next()) { String schemaName = schemas.getString("TABLE_SCHEM"); @@ -185,58 +177,40 @@ public class SqlCompleter { return res; } + + private static void fillTableNames(String schema, DatabaseMetaData meta, Set tables) { + try (ResultSet tbls = meta.getTables(schema, schema, "%", null)) { + while (tbls.next()) { + String table = tbls.getString("TABLE_NAME"); + tables.add(table); + } + } catch (Throwable t) { + logger.error("Failed to retrieve the table name", t); + } + } + /** * Fill two map with list of tables and list of columns * - * @param catalogName name of a catalog - * @param meta metadata from connection to database - * @param schemaFilter a schema name pattern; must match the schema name - * as it is stored in the database; "" retrieves those without a schema; - * null means that the schema name should not be used to narrow - * the search; supports '%'; for example "prod_v_%" - * @param tables function fills this map, for every schema name adds - * set of table names within the schema - * @param columns function fills this map, for every table name adds set + * @param schema name of a scheme + * @param table name of a table + * @param meta meta metadata from connection to database + * @param columns function fills this set, for every table name adds set * of columns within the table; table name is in format schema_name.table_name */ - private static void fillTableAndColumnNames(String catalogName, DatabaseMetaData meta, - String schemaFilter, - Map> tables, - Map> columns) { - try { - ResultSet cols = meta.getColumns(catalogName, StringUtils.EMPTY, "%", "%"); - try { - while (cols.next()) { - String schema = cols.getString("TABLE_SCHEM"); - if (schema == null) { - schema = cols.getString("TABLE_CAT"); - } - if (!schemaFilter.equals("") && !schema.matches(schemaFilter.replace("%", ".*?"))) { - continue; - } - String table = cols.getString("TABLE_NAME"); - String column = cols.getString("COLUMN_NAME"); - if (!isBlank(table)) { - String schemaTable = schema + "." + table; - if (!columns.containsKey(schemaTable)) { - columns.put(schemaTable, new HashSet()); - } - columns.get(schemaTable).add(column); - if (!tables.containsKey(schema)) { - tables.put(schema, new HashSet()); - } - tables.get(schema).add(table); - } - } - } finally { - cols.close(); + private static void fillColumnNames(String schema, String table, DatabaseMetaData meta, + Set columns) { + try (ResultSet cols = meta.getColumns(schema, schema, table, "%")) { + while (cols.next()) { + String column = cols.getString("COLUMN_NAME"); + columns.add(column); } } catch (Throwable t) { logger.error("Failed to retrieve the column name", t); } } - public static Set getSqlKeywordsCompletions(Connection connection) throws IOException, + public static Set getSqlKeywordsCompletions(DatabaseMetaData meta) throws IOException, SQLException { // Add the default SQL completions @@ -246,12 +220,11 @@ public class SqlCompleter { Set completions = new TreeSet<>(); - if (null != connection) { - DatabaseMetaData metaData = connection.getMetaData(); + if (null != meta) { // Add the driver specific SQL completions String driverSpecificKeywords = - "/" + metaData.getDriverName().replace(" ", "-").toLowerCase() + "-sql.keywords"; + "/" + meta.getDriverName().replace(" ", "-").toLowerCase() + "-sql.keywords"; logger.info("JDBC DriverName:" + driverSpecificKeywords); try { if (SqlCompleter.class.getResource(driverSpecificKeywords) != null) { @@ -269,27 +242,27 @@ public class SqlCompleter { // Add the keywords from the current JDBC connection try { - keywords += "," + metaData.getSQLKeywords(); + keywords += "," + meta.getSQLKeywords(); } catch (Exception e) { logger.debug("fail to get SQL key words from database metadata: " + e, e); } try { - keywords += "," + metaData.getStringFunctions(); + keywords += "," + meta.getStringFunctions(); } catch (Exception e) { logger.debug("fail to get string function names from database metadata: " + e, e); } try { - keywords += "," + metaData.getNumericFunctions(); + keywords += "," + meta.getNumericFunctions(); } catch (Exception e) { logger.debug("fail to get numeric function names from database metadata: " + e, e); } try { - keywords += "," + metaData.getSystemFunctions(); + keywords += "," + meta.getSystemFunctions(); } catch (Exception e) { logger.debug("fail to get system function names from database metadata: " + e, e); } try { - keywords += "," + metaData.getTimeDateFunctions(); + keywords += "," + meta.getTimeDateFunctions(); } catch (Exception e) { logger.debug("fail to get time date function names from database metadata: " + e, e); } @@ -307,95 +280,101 @@ public class SqlCompleter { return completions; } - /** - * Initializes local schema completers from list of schema names - * - * @param schemas set of schema names - */ - private void initSchemas(Set schemas) { - schemasCompleter = new StringsCompleter(new TreeSet<>(schemas)); - } - - /** - * Initializes local table completers from list of table name - * - * @param tables for every schema name there is a set of table names within the schema - */ - private void initTables(Map> tables) { - tablesCompleters.clear(); - for (Map.Entry> entry : tables.entrySet()) { - tablesCompleters.put(entry.getKey(), new StringsCompleter(new TreeSet<>(entry.getValue()))); - } - } - - /** - * Initializes local column completers from list of column names - * - * @param columns for every table name there is a set of columns within the table; - * table name is in format schema_name.table_name - */ - private void initColumns(Map> columns) { - columnsCompleters.clear(); - for (Map.Entry> entry : columns.entrySet()) { - columnsCompleters.put(entry.getKey(), new StringsCompleter(new TreeSet<>(entry.getValue()))); - } - } - - /** - * Initializes all local completers - * - * @param schemas set of schema names - * @param tables for every schema name there is a set of table names within the schema - * @param columns for every table name there is a set of columns within the table; - * table name is in format schema_name.table_name - * @param keywords set with sql keywords - */ - public void init(Set schemas, Map> tables, - Map> columns, Set keywords) { - initSchemas(schemas); - initTables(tables); - initColumns(columns); - keywordCompleter = new StringsCompleter(keywords); - } - /** * Initializes all local completers from database connection * * @param connection database connection - * @param schemaFiltersString a comma separated schema name patterns; supports '%' symbol; - * for example "prod_v_%,prod_t_%" + * @param schemaFiltersString a comma separated schema name patterns, supports '%' symbol; + * for example "prod_v_%,prod_t_%" */ - public void initFromConnection(Connection connection, String schemaFiltersString) { - if (schemaFiltersString == null) { - schemaFiltersString = StringUtils.EMPTY; - } - List schemaFilters = Arrays.asList(schemaFiltersString.split(",")); - + public void createOrUpdateFromConnection(Connection connection, String schemaFiltersString, + String buffer, int cursor) { try (Connection c = connection) { - Map> tables = new HashMap<>(); - Map> columns = new HashMap<>(); + if (schemaFiltersString == null) { + schemaFiltersString = StringUtils.EMPTY; + } + List schemaFilters = Arrays.asList(schemaFiltersString.split(",")); + CursorArgument cursorArgument = parseCursorArgument(buffer, cursor); + + Set tables = new HashSet<>(); + Set columns = new HashSet<>(); Set schemas = new HashSet<>(); Set catalogs = new HashSet<>(); - Set keywords = getSqlKeywordsCompletions(connection); - if (connection != null) { - schemas = getSchemaNames(connection.getMetaData(), schemaFilters); - catalogs = getCatalogNames(connection.getMetaData(), schemaFilters); - if (schemas.size() == 0) { - schemas.addAll(catalogs); + Set keywords = new HashSet<>(); + + if (c != null) { + DatabaseMetaData databaseMetaData = c.getMetaData(); + if (keywordCompleter == null || keywordCompleter.getCompleter() == null + || keywordCompleter.isExpired()) { + keywords = getSqlKeywordsCompletions(databaseMetaData); + initKeywords(keywords); } - for (String schema : schemas) { - for (String schemaFilter : schemaFilters) { - fillTableAndColumnNames(schema, connection.getMetaData(), schemaFilter, tables, - columns); + if (cursorArgument.needLoadSchemas() && + (schemasCompleter == null || schemasCompleter.getCompleter() == null + || schemasCompleter.isExpired())) { + schemas = getSchemaNames(databaseMetaData, schemaFilters); + catalogs = getCatalogNames(databaseMetaData, schemaFilters); + + if (schemas.size() == 0) { + schemas.addAll(catalogs); } + + initSchemas(schemas); } + + CachedCompleter tablesCompleter = tablesCompleters.get(cursorArgument.getSchema()); + if (cursorArgument.needLoadTables() && + (tablesCompleter == null || tablesCompleter.isExpired())) { + fillTableNames(cursorArgument.getSchema(), databaseMetaData, tables); + initTables(cursorArgument.getSchema(), tables); + } + + String schemaTable = + String.format("%s.%s", cursorArgument.getSchema(), cursorArgument.getTable()); + CachedCompleter columnsCompleter = columnsCompleters.get(schemaTable); + + if (cursorArgument.needLoadColumns() && + (columnsCompleter == null || columnsCompleter.isExpired())) { + fillColumnNames(cursorArgument.getSchema(), cursorArgument.getTable(), databaseMetaData, + columns); + initColumns(schemaTable, columns); + } + + logger.info("Completer initialized with " + schemas.size() + " schemas, " + + columns.size() + " tables and " + keywords.size() + " keywords"); } - init(schemas, tables, columns, keywords); - logger.info("Completer initialized with " + schemas.size() + " schemas, " + - columns.size() + " tables and " + keywords.size() + " keywords"); } catch (SQLException | IOException e) { - logger.error("Failed to update the metadata conmpletions", e); + logger.error("Failed to update the metadata completions", e); + } + } + + + + public void initKeywords(Set keywords) { + if (keywords != null && !keywords.isEmpty()) { + keywordCompleter = new CachedCompleter(new StringsCompleter(keywords), 0); + } + } + + public void initSchemas(Set schemas) { + if (schemas != null && !schemas.isEmpty()) { + schemasCompleter = new CachedCompleter( + new StringsCompleter(new TreeSet<>(schemas)), ttlInSeconds); + } + } + + public void initTables(String schema, Set tables) { + if (tables != null && !tables.isEmpty()) { + tablesCompleters.put(schema, new CachedCompleter( + new StringsCompleter(new TreeSet<>(tables)), ttlInSeconds)); + } + } + + public void initColumns(String schemaTable, Set columns) { + if (columns != null && !columns.isEmpty()) { + columnsCompleters.put(schemaTable, + new CachedCompleter(new StringsCompleter(columns), ttlInSeconds)); } } @@ -422,7 +401,7 @@ public class SqlCompleter { * @return -1 in case of no candidates found, 0 otherwise */ private int completeKeyword(String buffer, int cursor, List candidates) { - return keywordCompleter.complete(buffer, cursor, candidates); + return keywordCompleter.getCompleter().complete(buffer, cursor, candidates); } /** @@ -431,7 +410,7 @@ public class SqlCompleter { * @return -1 in case of no candidates found, 0 otherwise */ private int completeSchema(String buffer, int cursor, List candidates) { - return schemasCompleter.complete(buffer, cursor, candidates); + return schemasCompleter.getCompleter().complete(buffer, cursor, candidates); } /** @@ -441,21 +420,12 @@ public class SqlCompleter { */ private int completeTable(String schema, String buffer, int cursor, List candidates) { - if (schema == null) { - int res = -1; - Set candidatesSet = new HashSet<>(); - for (StringsCompleter stringsCompleter : tablesCompleters.values()) { - int resTable = stringsCompleter.complete(buffer, cursor, candidatesSet); - res = Math.max(res, resTable); - } - candidates.addAll(candidatesSet); - return res; - } // Wrong schema - if (!tablesCompleters.containsKey(schema) && schema != null) + if (schema == null || !tablesCompleters.containsKey(schema)) return -1; - else - return tablesCompleters.get(schema).complete(buffer, cursor, candidates); + else { + return tablesCompleters.get(schema).getCompleter().complete(buffer, cursor, candidates); + } } /** @@ -465,22 +435,12 @@ public class SqlCompleter { */ private int completeColumn(String schema, String table, String buffer, int cursor, List candidates) { - if (table == null && schema == null) { - int res = -1; - Set candidatesSet = new HashSet<>(); - for (StringsCompleter stringsCompleter : columnsCompleters.values()) { - int resColumn = stringsCompleter.complete(buffer, cursor, candidatesSet); - res = Math.max(res, resColumn); - } - candidates.addAll(candidatesSet); - return res; - } // Wrong schema or wrong table - if (!tablesCompleters.containsKey(schema) || - !columnsCompleters.containsKey(schema + "." + table)) { + if (schema == null || table == null || !columnsCompleters.containsKey(schema + "." + table)) { return -1; } else { - return columnsCompleters.get(schema + "." + table).complete(buffer, cursor, candidates); + return columnsCompleters.get(schema + "." + table).getCompleter() + .complete(buffer, cursor, candidates); } } @@ -489,74 +449,55 @@ public class SqlCompleter { * a schema, a table of a column or a keyword * * @param aliases for every alias contains table name in format schema_name.table_name - * @param isColumnAllowed if false the function will not search and complete columns * @return -1 in case of no candidates found, 0 otherwise */ public int completeName(String buffer, int cursor, List candidates, - Map aliases, boolean isColumnAllowed) { + Map aliases) { + CursorArgument cursorArgument = parseCursorArgument(buffer, cursor); - // points divide the name to the schema, table and column - find them - int pointPos1 = -1; - int pointPos2 = -1; - - if (StringUtils.isNotEmpty(buffer)) { - if (buffer.length() > cursor) { - buffer = buffer.substring(0, cursor + 1); - } - pointPos1 = buffer.indexOf('.'); - pointPos2 = buffer.indexOf('.', pointPos1 + 1); - } // find schema and table name if they are String schema; String table; String column; - if (pointPos1 == -1) { // process all + if (cursorArgument.getSchema() == null) { // process all List keywordsCandidates = new ArrayList(); List schemaCandidates = new ArrayList<>(); - List tableCandidates = new ArrayList<>(); - List columnCandidates = new ArrayList<>(); int keywordsRes = completeKeyword(buffer, cursor, keywordsCandidates); int schemaRes = completeSchema(buffer, cursor, schemaCandidates); - int tableRes = completeTable(null, buffer, cursor, tableCandidates); - int columnRes = -1; - if (isColumnAllowed) { - columnRes = completeColumn(null, null, buffer, cursor, columnCandidates); - } addCompletions(candidates, keywordsCandidates, CompletionType.keyword.name()); addCompletions(candidates, schemaCandidates, CompletionType.schema.name()); - addCompletions(candidates, tableCandidates, CompletionType.table.name()); - addCompletions(candidates, columnCandidates, CompletionType.column.name()); - return NumberUtils.max(new int[]{keywordsRes, schemaRes, tableRes, columnRes}); + return NumberUtils.max(new int[]{keywordsRes, schemaRes}); } else { - schema = buffer.substring(0, pointPos1); + schema = cursorArgument.getSchema(); if (aliases.containsKey(schema)) { // process alias case String alias = aliases.get(schema); int pointPos = alias.indexOf('.'); schema = alias.substring(0, pointPos); table = alias.substring(pointPos + 1); - column = buffer.substring(pointPos1 + 1); - } else if (pointPos2 == -1) { // process schema.table case + column = cursorArgument.getColumn(); + List columnCandidates = new ArrayList(); + int columnRes = completeColumn(schema, table, column, cursorArgument.getCursorPosition(), + columnCandidates); + addCompletions(candidates, columnCandidates, CompletionType.column.name()); + // process schema.table case + } else if (cursorArgument.getTable() != null && cursorArgument.getColumn() == null) { List tableCandidates = new ArrayList(); - table = buffer.substring(pointPos1 + 1); - int tableRes = completeTable(schema, table, cursor - pointPos1 - 1, tableCandidates); + table = cursorArgument.getTable(); + int tableRes = completeTable(schema, table, cursorArgument.getCursorPosition(), + tableCandidates); addCompletions(candidates, tableCandidates, CompletionType.table.name()); return tableRes; } else { - table = buffer.substring(pointPos1 + 1, pointPos2); - column = buffer.substring(pointPos2 + 1); + List columnCandidates = new ArrayList(); + table = cursorArgument.getTable(); + column = cursorArgument.getColumn(); + int columnRes = completeColumn(schema, table, column, cursorArgument.getCursorPosition(), + columnCandidates); + addCompletions(candidates, columnCandidates, CompletionType.column.name()); } } - // here in case of column - if (table != null && isColumnAllowed) { - List columnCandidates = new ArrayList(); - int columnRes = completeColumn(schema, table, column, cursor - pointPos2 - 1, - columnCandidates); - addCompletions(candidates, columnCandidates, CompletionType.column.name()); - return columnRes; - } - return -1; } @@ -572,4 +513,92 @@ public class SqlCompleter { candidate.toString(), meta)); } } + + private CursorArgument parseCursorArgument(String buffer, int cursor) { + CursorArgument result = new CursorArgument(); + if (buffer != null && buffer.length() >= cursor) { + String buf = buffer.substring(0, cursor); + if (StringUtils.isNotBlank(buf)) { + ArgumentList argumentList = sqlDelimiter.delimit(buf, cursor); + String cursorArgument = argumentList.getCursorArgument(); + if (cursorArgument != null) { + int pointPos1 = cursorArgument.indexOf('.'); + int pointPos2 = cursorArgument.indexOf('.', pointPos1 + 1); + if (pointPos1 > -1) { + result.setSchema(cursorArgument.substring(0, pointPos1).trim()); + if (pointPos2 > -1) { + result.setTable(cursorArgument.substring(pointPos1 + 1, pointPos2)); + result.setColumn(cursorArgument.substring(pointPos2 + 1)); + result.setCursorPosition(cursor - pointPos2 - 1); + } else { + result.setTable(cursorArgument.substring(pointPos1 + 1)); + result.setCursorPosition(cursor - pointPos1 - 1); + } + } + } + } + } + + return result; + } + + private class CursorArgument { + private String schema; + private String table; + private String column; + private int cursorPosition; + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String getColumn() { + return column; + } + + public void setColumn(String column) { + this.column = column; + } + + public int getCursorPosition() { + return cursorPosition; + } + + public void setCursorPosition(int cursorPosition) { + this.cursorPosition = cursorPosition; + } + + public boolean needLoadSchemas() { + if (table == null && column == null) { + return true; + } + return false; + } + + public boolean needLoadTables() { + if (schema != null && table != null && column == null) { + return true; + } + return false; + } + + public boolean needLoadColumns() { + if (schema != null && table != null && column != null) { + return true; + } + return false; + } + } } diff --git a/jdbc/src/main/resources/interpreter-setting.json b/jdbc/src/main/resources/interpreter-setting.json index fb8b8b2a23..21ff685001 100644 --- a/jdbc/src/main/resources/interpreter-setting.json +++ b/jdbc/src/main/resources/interpreter-setting.json @@ -22,6 +22,12 @@ "defaultValue": "", "description": "The JDBC user password" }, + "default.completer.ttlInSeconds": { + "envName": null, + "propertyName": "default.completer.ttlInSeconds", + "defaultValue": "120", + "description": "Time to live sql completer in seconds (-1 to update everytime, 0 to disable update)" + }, "default.driver": { "envName": null, "propertyName": "default.driver", @@ -40,6 +46,12 @@ "defaultValue": "", "description": "SQL which executes while opening connection" }, + "default.splitQueries": { + "envName": null, + "propertyName": "default.splitQueries", + "defaultValue": "false", + "description": "Each query is executed apart and returns the result" + }, "common.max_count": { "envName": null, "propertyName": "common.max_count", diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java index 7c2eef39b0..e6f9598f8b 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/JDBCInterpreterTest.java @@ -27,8 +27,11 @@ import static org.junit.Assert.*; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.*; -import java.util.ArrayList; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; import java.util.Properties; @@ -87,7 +90,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { PreparedStatement insertStatement = connection.prepareStatement("insert into test_table(id, name) values ('a', 'a_name'),('b', 'b_name'),('c', ?);"); insertStatement.setString(1, null); insertStatement.execute(); - interpreterContext = new InterpreterContext("", "1", null, "", "", new AuthenticationInfo(), null, null, null, null, + interpreterContext = new InterpreterContext("", "1", null, "", "", new AuthenticationInfo("testUser"), null, null, null, null, null, null); } @@ -175,21 +178,34 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { String sqlQuery = "insert into test_table(id, name) values ('a', ';\"');" + "select * from test_table;" + "select * from test_table WHERE ID = \";'\";" + - "select * from test_table WHERE ID = ';'"; + "select * from test_table WHERE ID = ';';" + + "select '\n', ';';" + + "select replace('A\\;B', '\\', 'text');" + + "select '\\', ';';" + + "select '''', ';'"; Properties properties = new Properties(); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); - ArrayList multipleSqlArray = t.splitSqlQueries(sqlQuery); - assertEquals(4, multipleSqlArray.size()); + List multipleSqlArray = t.splitSqlQueries(sqlQuery); + assertEquals(8, multipleSqlArray.size()); assertEquals("insert into test_table(id, name) values ('a', ';\"')", multipleSqlArray.get(0)); assertEquals("select * from test_table", multipleSqlArray.get(1)); assertEquals("select * from test_table WHERE ID = \";'\"", multipleSqlArray.get(2)); assertEquals("select * from test_table WHERE ID = ';'", multipleSqlArray.get(3)); + assertEquals("select '\n', ';'", multipleSqlArray.get(4)); + assertEquals("select replace('A\\;B', '\\', 'text')", multipleSqlArray.get(5)); + assertEquals("select '\\', ';'", multipleSqlArray.get(6)); + assertEquals("select '''', ';'", multipleSqlArray.get(7)); } @Test - public void testSelectMultipleQuries() throws SQLException, IOException { + public void testQueryWithEsсapedCharacters() throws SQLException, IOException { + String sqlQuery = "select '\\n', ';';" + + "select replace('A\\;B', '\\', 'text');" + + "select '\\', ';';" + + "select '''', ';'"; + Properties properties = new Properties(); properties.setProperty("common.max_count", "1000"); properties.setProperty("common.max_retry", "3"); @@ -197,6 +213,34 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { properties.setProperty("default.url", getJdbcConnection()); properties.setProperty("default.user", ""); properties.setProperty("default.password", ""); + properties.setProperty("default.splitQueries", "true"); + JDBCInterpreter t = new JDBCInterpreter(properties); + t.open(); + + InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext); + + assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code()); + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(1).getType()); + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(2).getType()); + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(3).getType()); + assertEquals("'\\n'\t';'\n\\n\t;\n", interpreterResult.message().get(0).getData()); + assertEquals("'Atext;B'\nAtext;B\n", interpreterResult.message().get(1).getData()); + assertEquals("'\\'\t';'\n\\\t;\n", interpreterResult.message().get(2).getData()); + assertEquals("''''\t';'\n'\t;\n", interpreterResult.message().get(3).getData()); + + } + + @Test + public void testSelectMultipleQueries() throws SQLException, IOException { + Properties properties = new Properties(); + properties.setProperty("common.max_count", "1000"); + properties.setProperty("common.max_retry", "3"); + properties.setProperty("default.driver", "org.h2.Driver"); + properties.setProperty("default.url", getJdbcConnection()); + properties.setProperty("default.user", ""); + properties.setProperty("default.password", ""); + properties.setProperty("default.splitQueries", "true"); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); @@ -213,6 +257,28 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { assertEquals("ID\tNAME\n", interpreterResult.message().get(1).getData()); } + @Test + public void testDefaultSplitQuries() throws SQLException, IOException { + Properties properties = new Properties(); + properties.setProperty("common.max_count", "1000"); + properties.setProperty("common.max_retry", "3"); + properties.setProperty("default.driver", "org.h2.Driver"); + properties.setProperty("default.url", getJdbcConnection()); + properties.setProperty("default.user", ""); + properties.setProperty("default.password", ""); + JDBCInterpreter t = new JDBCInterpreter(properties); + t.open(); + + String sqlQuery = "select * from test_table;" + + "select * from test_table WHERE ID = ';';"; + InterpreterResult interpreterResult = t.interpret(sqlQuery, interpreterContext); + assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code()); + assertEquals(1, interpreterResult.message().size()); + + assertEquals(InterpreterResult.Type.TABLE, interpreterResult.message().get(0).getType()); + assertEquals("ID\tNAME\na\ta_name\nb\tb_name\nc\tnull\n", interpreterResult.message().get(0).getData()); + } + @Test public void testSelectQueryWithNull() throws SQLException, IOException { Properties properties = new Properties(); @@ -296,9 +362,9 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { jdbcInterpreter.interpret("", interpreterContext); - List completionList = jdbcInterpreter.completion("sel", 3, null); + List completionList = jdbcInterpreter.completion("sel", 3, interpreterContext); - InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select ", "select ", CompletionType.keyword.name()); + InterpreterCompletion correctCompletionKeyword = new InterpreterCompletion("select", "select", CompletionType.keyword.name()); assertEquals(1, completionList.size()); assertEquals(true, completionList.contains(correctCompletionKeyword)); @@ -465,6 +531,7 @@ public class JDBCInterpreterTest extends BasicJDBCTestCaseAdapter { properties.setProperty("default.url", getJdbcConnection()); properties.setProperty("default.user", ""); properties.setProperty("default.password", ""); + properties.setProperty("default.splitQueries", "true"); JDBCInterpreter t = new JDBCInterpreter(properties); t.open(); diff --git a/jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java b/jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java index 999f7de62b..19150ccebc 100644 --- a/jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java +++ b/jdbc/src/test/java/org/apache/zeppelin/jdbc/SqlCompleterTest.java @@ -86,7 +86,7 @@ public class SqlCompleterTest { private void expectedCompletions(String buffer, int cursor, Set expected) { if (StringUtils.isNotEmpty(buffer) && buffer.length() > cursor) { - buffer = buffer.substring(0, cursor + 1); + buffer = buffer.substring(0, cursor); } List candidates = new ArrayList<>(); @@ -140,13 +140,10 @@ public class SqlCompleterTest { private ArgumentCompleter.WhitespaceArgumentDelimiter delimiter = new ArgumentCompleter.WhitespaceArgumentDelimiter(); - private SqlCompleter sqlCompleter = new SqlCompleter(); + private SqlCompleter sqlCompleter = new SqlCompleter(0); @Before public void beforeTest() throws IOException, SQLException { - - Map> tables = new HashMap<>(); - Map> columns = new HashMap<>(); Set schemas = new HashSet<>(); Set keywords = new HashSet<>(); @@ -158,37 +155,42 @@ public class SqlCompleterTest { keywords.add("LIMIT"); keywords.add("FROM"); + sqlCompleter.initKeywords(keywords); + schemas.add("prod_dds"); schemas.add("prod_emart"); + sqlCompleter.initSchemas(schemas); + Set prod_dds_tables = new HashSet<>(); prod_dds_tables.add("financial_account"); prod_dds_tables.add("customer"); + sqlCompleter.initTables("prod_dds", prod_dds_tables); + Set prod_emart_tables = new HashSet<>(); prod_emart_tables.add("financial_account"); - tables.put("prod_dds", prod_dds_tables); - tables.put("prod_emart", prod_emart_tables); + sqlCompleter.initTables("prod_emart", prod_emart_tables); Set prod_dds_financial_account_columns = new HashSet<>(); prod_dds_financial_account_columns.add("account_rk"); prod_dds_financial_account_columns.add("account_id"); + sqlCompleter.initColumns("prod_dds.financial_account", prod_dds_financial_account_columns); + Set prod_dds_customer_columns = new HashSet<>(); prod_dds_customer_columns.add("customer_rk"); prod_dds_customer_columns.add("name"); prod_dds_customer_columns.add("birth_dt"); + sqlCompleter.initColumns("prod_dds.customer", prod_dds_customer_columns); + Set prod_emart_financial_account_columns = new HashSet<>(); prod_emart_financial_account_columns.add("account_rk"); prod_emart_financial_account_columns.add("balance_amt"); - columns.put("prod_dds.financial_account", prod_dds_financial_account_columns); - columns.put("prod_dds.customer", prod_dds_customer_columns); - columns.put("prod_emart.financial_account", prod_emart_financial_account_columns); - - sqlCompleter.init(schemas, tables, columns, keywords); + sqlCompleter.initColumns("prod_emart.financial_account", prod_emart_financial_account_columns); tester = new CompleterTester(sqlCompleter); } @@ -223,8 +225,8 @@ public class SqlCompleterTest { int cursor = 0; List candidates = new ArrayList<>(); Map aliases = new HashMap<>(); - sqlCompleter.completeName(buffer, cursor, candidates, aliases, true); - assertEquals(17, candidates.size()); + sqlCompleter.completeName(buffer, cursor, candidates, aliases); + assertEquals(9, candidates.size()); assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()))); assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()))); assertTrue(candidates.contains(new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()))); @@ -234,14 +236,6 @@ public class SqlCompleterTest { assertTrue(candidates.contains(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()))); assertTrue(candidates.contains(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()))); assertTrue(candidates.contains(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("customer", "customer", CompletionType.table.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("name", "name", CompletionType.column.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name()))); - assertTrue(candidates.contains(new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name()))); } @Test @@ -250,7 +244,7 @@ public class SqlCompleterTest { int cursor = 3; List candidates = new ArrayList<>(); Map aliases = new HashMap<>(); - sqlCompleter.completeName(buffer, cursor, candidates, aliases, false); + sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(2, candidates.size()); assertTrue(candidates.contains(new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()))); assertTrue(candidates.contains(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()))); @@ -262,7 +256,7 @@ public class SqlCompleterTest { int cursor = 11; List candidates = new ArrayList<>(); Map aliases = new HashMap<>(); - sqlCompleter.completeName(buffer, cursor, candidates, aliases, false); + sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(1, candidates.size()); assertTrue(candidates.contains(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()))); } @@ -273,7 +267,7 @@ public class SqlCompleterTest { int cursor = 30; List candidates = new ArrayList<>(); Map aliases = new HashMap<>(); - sqlCompleter.completeName(buffer, cursor, candidates, aliases, true); + sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(2, candidates.size()); assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()))); assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()))); @@ -286,7 +280,7 @@ public class SqlCompleterTest { List candidates = new ArrayList<>(); Map aliases = new HashMap<>(); aliases.put("a", "prod_dds.financial_account"); - sqlCompleter.completeName(buffer, cursor, candidates, aliases, true); + sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(2, candidates.size()); assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()))); assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()))); @@ -299,7 +293,7 @@ public class SqlCompleterTest { List candidates = new ArrayList<>(); Map aliases = new HashMap<>(); aliases.put("a", "prod_dds.financial_account"); - sqlCompleter.completeName(buffer, cursor, candidates, aliases, true); + sqlCompleter.completeName(buffer, cursor, candidates, aliases); assertEquals(2, candidates.size()); assertTrue(candidates.contains(new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()))); assertTrue(candidates.contains(new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()))); @@ -308,14 +302,14 @@ public class SqlCompleterTest { @Test public void testSchemaAndTable() { String buffer = "select * from prod_emart.fi"; - tester.buffer(buffer).from(19).to(23).expect(newHashSet(new InterpreterCompletion("prod_emart ", "prod_emart ", CompletionType.schema.name()))).test(); - tester.buffer(buffer).from(25).to(27).expect(newHashSet(new InterpreterCompletion("financial_account ", "financial_account ", CompletionType.table.name()))).test(); + tester.buffer(buffer).from(20).to(23).expect(newHashSet(new InterpreterCompletion("prod_emart", "prod_emart", CompletionType.schema.name()))).test(); + tester.buffer(buffer).from(25).to(27).expect(newHashSet(new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()))).test(); } @Test public void testEdges() { String buffer = " ORDER "; - tester.buffer(buffer).from(2).to(6).expect(newHashSet(new InterpreterCompletion("ORDER ", "ORDER ", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(3).to(7).expect(newHashSet(new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()))).test(); tester.buffer(buffer).from(0).to(1).expect(newHashSet( new InterpreterCompletion("ORDER", "ORDER", CompletionType.keyword.name()), new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()), @@ -325,37 +319,29 @@ public class SqlCompleterTest { new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()), new InterpreterCompletion("prod_dds", "prod_dds", CompletionType.schema.name()), new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()), - new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()), - new InterpreterCompletion("financial_account", "financial_account", CompletionType.table.name()), - new InterpreterCompletion("customer", "customer", CompletionType.table.name()), - new InterpreterCompletion("account_rk", "account_rk", CompletionType.column.name()), - new InterpreterCompletion("account_id", "account_id", CompletionType.column.name()), - new InterpreterCompletion("customer_rk", "customer_rk", CompletionType.column.name()), - new InterpreterCompletion("name", "name", CompletionType.column.name()), - new InterpreterCompletion("birth_dt", "birth_dt", CompletionType.column.name()), - new InterpreterCompletion("balance_amt", "balance_amt", CompletionType.column.name()) + new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()) )).test(); } @Test public void testMultipleWords() { String buffer = "SELE FRO LIM"; - tester.buffer(buffer).from(1).to(3).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test(); - tester.buffer(buffer).from(6).to(7).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test(); - tester.buffer(buffer).from(9).to(12).expect(newHashSet(new InterpreterCompletion("LIMIT ", "LIMIT ", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(2).to(4).expect(newHashSet(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(6).to(8).expect(newHashSet(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(10).to(12).expect(newHashSet(new InterpreterCompletion("LIMIT", "LIMIT", CompletionType.keyword.name()))).test(); } @Test public void testMultiLineBuffer() { String buffer = " \n SELE\nFRO"; - tester.buffer(buffer).from(4).to(6).expect(newHashSet(new InterpreterCompletion("SELECT ", "SELECT ", CompletionType.keyword.name()))).test(); - tester.buffer(buffer).from(9).to(11).expect(newHashSet(new InterpreterCompletion("FROM ", "FROM ", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(5).to(7).expect(newHashSet(new InterpreterCompletion("SELECT", "SELECT", CompletionType.keyword.name()))).test(); + tester.buffer(buffer).from(9).to(11).expect(newHashSet(new InterpreterCompletion("FROM", "FROM", CompletionType.keyword.name()))).test(); } @Test public void testMultipleCompletionSuggestions() { String buffer = "SU"; - tester.buffer(buffer).from(1).to(2).expect(newHashSet( + tester.buffer(buffer).from(2).to(2).expect(newHashSet( new InterpreterCompletion("SUBCLASS_ORIGIN", "SUBCLASS_ORIGIN", CompletionType.keyword.name()), new InterpreterCompletion("SUM", "SUM", CompletionType.keyword.name()), new InterpreterCompletion("SUBSTRING", "SUBSTRING", CompletionType.keyword.name())) diff --git a/livy/pom.xml b/livy/pom.xml index 3c121fcefb..5f9dec7226 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -37,14 +37,10 @@ 1.3 - 4.3.4 4.3.0.RELEASE 1.0.1.RELEASE - 3.2.4-Zeppelin - 1.7.0 - 1.9.5 0.3.0 2.1.0 2.6.0 @@ -80,7 +76,6 @@ org.apache.httpcomponents httpclient - ${httpcomponents.client.version} @@ -106,19 +101,6 @@ test - - org.assertj - assertj-core - ${assertj.version} - test - - - org.mockito - mockito-all - ${mockito.version} - test - - com.cloudera.livy livy-integration-test diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index b52ba16e57..523dfad881 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -21,9 +21,20 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang.StringUtils; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.*; @@ -38,11 +49,11 @@ import org.springframework.security.kerberos.client.KerberosRestTemplate; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; - import javax.net.ssl.SSLContext; import java.io.FileInputStream; import java.io.IOException; import java.security.KeyStore; +import java.security.Principal; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -52,7 +63,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - /** * Base class for livy interpreters. */ @@ -331,7 +341,17 @@ public abstract class BaseLivyInterpreter extends Interpreter { private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo, boolean displayAppInfo) { if (stmtInfo.output != null && stmtInfo.output.isError()) { - return new InterpreterResult(InterpreterResult.Code.ERROR, stmtInfo.output.evalue); + InterpreterResult result = new InterpreterResult(InterpreterResult.Code.ERROR); + StringBuilder sb = new StringBuilder(); + sb.append(stmtInfo.output.evalue); + // in case evalue doesn't have newline char + if (!stmtInfo.output.evalue.contains("\n")) + sb.append("\n"); + if (stmtInfo.output.traceback != null) { + sb.append(StringUtils.join(stmtInfo.output.traceback)); + } + result.add(sb.toString()); + return result; } else if (stmtInfo.isCancelled()) { // corner case, output might be null if it is cancelled. return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled"); @@ -407,6 +427,11 @@ public abstract class BaseLivyInterpreter extends Interpreter { private RestTemplate createRestTemplate() { + String keytabLocation = property.getProperty("zeppelin.livy.keytab"); + String principal = property.getProperty("zeppelin.livy.principal"); + boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) && + StringUtils.isNotEmpty(principal); + HttpClient httpClient = null; if (livyURL.startsWith("https:")) { String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore"); @@ -427,7 +452,37 @@ public abstract class BaseLivyInterpreter extends Interpreter { .loadTrustMaterial(trustStore) .build(); SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext); - httpClient = HttpClients.custom().setSSLSocketFactory(csf).build(); + HttpClientBuilder httpClientBuilder = HttpClients.custom().setSSLSocketFactory(csf); + RequestConfig reqConfig = new RequestConfig() { + @Override + public boolean isAuthenticationEnabled() { + return true; + } + }; + httpClientBuilder.setDefaultRequestConfig(reqConfig); + Credentials credentials = new Credentials() { + @Override + public String getPassword() { + return null; + } + + @Override + public Principal getUserPrincipal() { + return null; + } + }; + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(AuthScope.ANY, credentials); + httpClientBuilder.setDefaultCredentialsProvider(credsProvider); + if (isSpnegoEnabled) { + Registry authSchemeProviderRegistry = + RegistryBuilder.create() + .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()) + .build(); + httpClientBuilder.setDefaultAuthSchemeRegistry(authSchemeProviderRegistry); + } + + httpClient = httpClientBuilder.build(); } catch (Exception e) { throw new RuntimeException("Failed to create SSL HttpClient", e); } finally { @@ -441,9 +496,8 @@ public abstract class BaseLivyInterpreter extends Interpreter { } } - String keytabLocation = property.getProperty("zeppelin.livy.keytab"); - String principal = property.getProperty("zeppelin.livy.principal"); - if (StringUtils.isNotEmpty(keytabLocation) && StringUtils.isNotEmpty(principal)) { + + if (isSpnegoEnabled) { if (httpClient == null) { return new KerberosRestTemplate(keytabLocation, principal); } else { @@ -615,7 +669,18 @@ public abstract class BaseLivyInterpreter extends Interpreter { } public static StatementInfo fromJson(String json) { - return gson.fromJson(json, StatementInfo.class); + String right_json = ""; + try { + gson.fromJson(json, StatementInfo.class); + right_json = json; + } catch (Exception e) { + if (json.contains("\"traceback\":{}")) { + LOGGER.debug("traceback type mismatch, replacing the mismatching part "); + right_json = json.replace("\"traceback\":{}", "\"traceback\":[]"); + LOGGER.debug("new json string is {}", right_json); + } + } + return gson.fromJson(right_json, StatementInfo.class); } public boolean isAvailable() { @@ -632,7 +697,7 @@ public abstract class BaseLivyInterpreter extends Interpreter { public Data data; public String ename; public String evalue; - public Object traceback; + public String[] traceback; public TableMagic tableMagic; public boolean isError() { diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index d132b5b41d..20d044811b 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -239,6 +239,15 @@ public class LivySparkSQLInterpreter extends BaseLivyInterpreter { this.sparkInterpreter.close(); } + @Override + public int getProgress(InterpreterContext context) { + if (this.sparkInterpreter != null) { + return this.sparkInterpreter.getProgress(context); + } else { + return 0; + } + } + @Override protected String extractAppId() throws LivyException { // it wont' be called because it would delegate to LivySparkInterpreter diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 6db75e9dd7..824eac36c3 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -308,6 +308,8 @@ public class LivyInterpreterIT { assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(InterpreterResult.Type.TABLE, result.message().get(0).getType()); assertTrue(result.message().get(0).getData().contains("tableName")); + int r = sqlInterpreter.getProgress(context); + assertTrue(r == 0); } finally { sqlInterpreter.close(); } @@ -521,7 +523,7 @@ public class LivyInterpreterIT { } @Test - public void testPySparkInterpreter() { + public void testPySparkInterpreter() throws LivyException { if (!checkPreCondition()) { return; } @@ -534,6 +536,24 @@ public class LivyInterpreterIT { "title", "text", authInfo, null, null, null, null, null, output); pysparkInterpreter.open(); + // test traceback msg + try { + pysparkInterpreter.getLivyVersion(); + // for livy version >=0.3 , input some erroneous spark code, check the shown result is more than one line + InterpreterResult result = pysparkInterpreter.interpret("sc.parallelize(wrongSyntax(1, 2)).count()", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertTrue(result.message().get(0).getData().split("\n").length>1); + assertTrue(result.message().get(0).getData().contains("Traceback")); + } catch (APINotFoundException e) { + // only livy 0.2 can throw this exception since it doesn't have /version endpoint + // in livy 0.2, most error msg is encapsulated in evalue field, only print(a) in pyspark would return none-empty + // traceback + InterpreterResult result = pysparkInterpreter.interpret("print(a)", context); + assertEquals(InterpreterResult.Code.ERROR, result.code()); + assertTrue(result.message().get(0).getData().split("\n").length>1); + assertTrue(result.message().get(0).getData().contains("Traceback")); + } + try { InterpreterResult result = pysparkInterpreter.interpret("sc.version", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); @@ -761,6 +781,7 @@ public class LivyInterpreterIT { } } + private boolean isSpark2(BaseLivyInterpreter interpreter, InterpreterContext context) { InterpreterResult result = null; if (interpreter instanceof LivySparkRInterpreter) { diff --git a/notebook/2A94M5J1Z/note.json b/notebook/2A94M5J1Z/note.json index 4f896e8fbe..6e8e06fe29 100644 --- a/notebook/2A94M5J1Z/note.json +++ b/notebook/2A94M5J1Z/note.json @@ -83,7 +83,7 @@ "msg": [ { "type": "TEXT", - "data": "\nimport org.apache.commons.io.IOUtils\n\nimport java.net.URL\n\nimport java.nio.charset.Charset\n\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[0] at parallelize at \u003cconsole\u003e:32\n\ndefined class Bank\n\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string ... 3 more fields]\n\nwarning: there were 1 deprecation warning(s); re-run with -deprecation for details\n" + "data": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[36] at parallelize at \u003cconsole\u003e:43\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string ... 3 more fields]\nwarning: there were 1 deprecation warning(s); re-run with -deprecation for details\n" } ] }, diff --git a/pom.xml b/pom.xml index 0eb29c55a1..ea9312b67d 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,7 @@ scio zeppelin-web zeppelin-server + zeppelin-jupyter zeppelin-distribution @@ -96,8 +97,8 @@ 0.2.1 15.0 9.2.15.v20160210 - 4.3.3 - 4.3.6 + 4.4.1 + 4.5.1 4.0.2 2.5 1.9 @@ -105,6 +106,7 @@ 2.4 3.2.1 1.1.1 + 1.3.1 1.3.2 @@ -134,6 +136,9 @@ 64m 512m + + + @@ -229,6 +234,12 @@ ${commons.logging.version} + + commons-cli + commons-cli + ${commons.cli.version} + + com.google.guava guava @@ -555,6 +566,9 @@ ${plugin.surefire.version} -Xmx2g -Xms1g -Dfile.encoding=UTF-8 + + ${tests.to.exclude} + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.8.0-SNAPSHOT + .. + + + zeppelin-jupyter + jar + 0.8.0-SNAPSHOT + Zeppelin: Jupyter Support + Jupyter support for Apache Zeppelin + + + + com.google.code.gson + gson + + + + org.danilopianini + gson-extras + + + + com.google.guava + guava + + + + commons-cli + commons-cli + + + + + junit + junit + + + diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java new file mode 100644 index 0000000000..eb7db20686 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.google.common.base.Joiner; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.typeadapters.RuntimeTypeAdapterFactory; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +import org.apache.zeppelin.jupyter.nbformat.Cell; +import org.apache.zeppelin.jupyter.nbformat.CodeCell; +import org.apache.zeppelin.jupyter.nbformat.DisplayData; +import org.apache.zeppelin.jupyter.nbformat.Error; +import org.apache.zeppelin.jupyter.nbformat.ExecuteResult; +import org.apache.zeppelin.jupyter.nbformat.HeadingCell; +import org.apache.zeppelin.jupyter.nbformat.MarkdownCell; +import org.apache.zeppelin.jupyter.nbformat.Nbformat; +import org.apache.zeppelin.jupyter.nbformat.Output; +import org.apache.zeppelin.jupyter.nbformat.RawCell; +import org.apache.zeppelin.jupyter.nbformat.Stream; +import org.apache.zeppelin.jupyter.zformat.Note; +import org.apache.zeppelin.jupyter.zformat.Paragraph; +import org.apache.zeppelin.jupyter.zformat.Result; +import org.apache.zeppelin.jupyter.zformat.TypeData; + +/** + * + */ +public class JupyterUtil { + + private static final String TEXT_PLAIN = "text/plain"; + private static final String IMAGE_PNG = "image/png"; + + private final RuntimeTypeAdapterFactory cellTypeFactory; + private final RuntimeTypeAdapterFactory outputTypeFactory; + + public JupyterUtil() { + this.cellTypeFactory = RuntimeTypeAdapterFactory.of(Cell.class, "cell_type") + .registerSubtype(MarkdownCell.class, "markdown").registerSubtype(CodeCell.class, "code") + .registerSubtype(RawCell.class, "raw").registerSubtype(HeadingCell.class, "heading"); + this.outputTypeFactory = RuntimeTypeAdapterFactory.of(Output.class, "output_type") + .registerSubtype(ExecuteResult.class, "execute_result") + .registerSubtype(DisplayData.class, "display_data").registerSubtype(Stream.class, "stream") + .registerSubtype(Error.class, "error"); + } + + public Nbformat getNbformat(Reader in) { + return getNbformat(in, new GsonBuilder()); + } + + public Nbformat getNbformat(Reader in, GsonBuilder gsonBuilder) { + return getGson(gsonBuilder).fromJson(in, Nbformat.class); + } + + public Note getNote(Reader in, String codeReplaced, String markdownReplaced) { + return getNote(in, new GsonBuilder(), codeReplaced, markdownReplaced); + } + + public Note getNote(Reader in, GsonBuilder gsonBuilder, String codeReplaced, + String markdownReplaced) { + return getNote(getNbformat(in, gsonBuilder), codeReplaced, markdownReplaced); + } + + public Note getNote(Nbformat nbformat, String codeReplaced, String markdownReplaced) { + Note note = new Note(); + + String name = nbformat.getMetadata().getTitle(); + if (null == name) { + name = "Note converted from Jupyter"; + } + note.setName(name); + + String lineSeparator = System.lineSeparator(); + Paragraph paragraph; + List paragraphs = new ArrayList<>(); + String interpreterName; + List typeDataList; + String type; + String result; + + for (Cell cell : nbformat.getCells()) { + paragraph = new Paragraph(); + typeDataList = new ArrayList<>(); + + if (cell instanceof CodeCell) { + interpreterName = codeReplaced; + for (Output output : ((CodeCell) cell).getOutputs()) { + TypeData typeData; + if (output instanceof Stream) { + type = TypeData.TEXT; + result = Joiner.on(lineSeparator).join(((Stream) output).getText()); + typeData = new TypeData(type, result); + typeDataList.add(typeData); + } else if (output instanceof ExecuteResult || output instanceof DisplayData) { + Map data = (output instanceof ExecuteResult) ? + ((ExecuteResult) output).getData() : + ((DisplayData) output).getData(); + for (Map.Entry datum : data.entrySet()) { + if (TEXT_PLAIN.equals(datum.getKey())) { + type = TypeData.TEXT; + result = Joiner.on(lineSeparator).join((List) datum.getValue()); + } else if (IMAGE_PNG.equals(datum.getKey())) { + type = TypeData.HTML; + result = makeHTML(((String) datum.getValue()).replace("\n", "")); + } else { + type = TypeData.TEXT; + result = datum.getValue().toString(); + } + typeData = new TypeData(type, result); + typeDataList.add(typeData); + } + } else { + // Error + Error error = (Error) output; + type = TypeData.TEXT; + result = + Joiner.on(lineSeparator).join(new String[] {error.getEname(), error.getEvalue()}); + typeData = new TypeData(type, result); + typeDataList.add(typeData); + } + } + } else if (cell instanceof MarkdownCell || cell instanceof HeadingCell) { + interpreterName = markdownReplaced; + } else { + interpreterName = ""; + } + + paragraph.setText( + interpreterName + lineSeparator + Joiner.on(lineSeparator).join(cell.getSource())); + paragraph.setResults(new Result(Result.SUCCESS, typeDataList)); + + paragraphs.add(paragraph); + } + + note.setParagraphs(paragraphs); + + return note; + } + + private Gson getGson(GsonBuilder gsonBuilder) { + return gsonBuilder.registerTypeAdapterFactory(cellTypeFactory) + .registerTypeAdapterFactory(outputTypeFactory).create(); + } + + private String makeHTML(String image) { + return "
    "; + } + + public static void main(String[] args) throws ParseException, IOException { + Options options = new Options(); + options.addOption("i", true, "Jupyter notebook file"); + options.addOption("o", true, "Zeppelin note file. Default: note.json"); + + CommandLineParser parser = new DefaultParser(); + CommandLine cmd = parser.parse(options, args); + + if (!cmd.hasOption("i")) { + new HelpFormatter().printHelp("java " + JupyterUtil.class.getName(), options); + System.exit(1); + } + + Path jupyterPath = Paths.get(cmd.getOptionValue("i")); + Path zeppelinPath = Paths.get(cmd.hasOption("o") ? cmd.getOptionValue("o") : "note.json"); + + try (BufferedReader in = new BufferedReader(new FileReader(jupyterPath.toFile())); + FileWriter fw = new FileWriter(zeppelinPath.toFile())) { + Note note = new JupyterUtil().getNote(in, "python", "md"); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + gson.toJson(note, fw); + } + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Author.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Author.java new file mode 100644 index 0000000000..c3e23e50ba --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Author.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; + +/** + * + */ +public class Author { + + @SerializedName("name") + private String name; + +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java new file mode 100644 index 0000000000..d44ec023f3 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public abstract class Cell { + + @SerializedName("cell_type") + private String cellType; + + @SerializedName("metadata") + private CellMetadata metadata; + + @SerializedName("source") + private List source; + + public String getCellType() { + return cellType; + } + + public CellMetadata getMetadata() { + return metadata; + } + + public List getSource() { + return source; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/CellMetadata.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/CellMetadata.java new file mode 100644 index 0000000000..a35a26239b --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/CellMetadata.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public class CellMetadata { + + @SerializedName("name") + private String name; + + @SerializedName("tags") + private List tags; + +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/CodeCell.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/CodeCell.java new file mode 100644 index 0000000000..973448c14a --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/CodeCell.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public class CodeCell extends Cell { + + @SerializedName("outputs") + private List outputs; + + public List getOutputs() { + return outputs; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java new file mode 100644 index 0000000000..68bfbfeb85 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.Map; + +/** + * + */ +public class DisplayData extends Output { + + @SerializedName("data") + private Map data; + + public Map getData() { + return data; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java new file mode 100644 index 0000000000..61466fa7cb --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public class Error extends Output { + @SerializedName("ename") + private String ename; + + @SerializedName("evalue") + private String evalue; + + @SerializedName("traceback") + private List traceback; + + public String getEname() { + return ename; + } + + public String getEvalue() { + return evalue; + } + + public List getTraceback() { + return traceback; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java new file mode 100644 index 0000000000..01f0cea047 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.Map; + +/** + * + */ +public class ExecuteResult extends Output { + + @SerializedName("execution_count") + private int executionCount; + + @SerializedName("data") + private Map data; + + public Map getData() { + return data; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/HeadingCell.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/HeadingCell.java new file mode 100644 index 0000000000..70070887bd --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/HeadingCell.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; + +/** + * + */ +public class HeadingCell extends Cell { + + @SerializedName("level") + private int level; + + public int getLevel() { + return level; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Kernelspec.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Kernelspec.java new file mode 100644 index 0000000000..6232416851 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Kernelspec.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; + +/** + * + */ +public class Kernelspec { + + @SerializedName("name") + private String name; + + @SerializedName("display_name") + private String displayName; +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/LanguageInfo.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/LanguageInfo.java new file mode 100644 index 0000000000..cc74089993 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/LanguageInfo.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; + +/** + * + */ +public class LanguageInfo { + + @SerializedName("name") + private String name; + + @SerializedName("codemirror_mode") + private Object codemirrorMode; + + @SerializedName("file_extension") + private String fileExtension; + + @SerializedName("mimetype") + private String mimetype; + + @SerializedName("pygments_lexer") + private String pygmentsLexer; + +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/MarkdownCell.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/MarkdownCell.java new file mode 100644 index 0000000000..a1b220bc96 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/MarkdownCell.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +/** + * + */ +public class MarkdownCell extends Cell { + +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Metadata.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Metadata.java new file mode 100644 index 0000000000..f2da8e4984 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Metadata.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public class Metadata { + + @SerializedName("kernelspec") + private Kernelspec kernelspec; + + @SerializedName("language_info") + private LanguageInfo languageInfo; + + @SerializedName("orig_nbformat") + private int origNbformat; + + @SerializedName("title") + private String title; + + @SerializedName("authors") + private List authors; + + public Kernelspec getKernelspec() { + return kernelspec; + } + + public LanguageInfo getLanguageInfo() { + return languageInfo; + } + + public int getOrigNbformat() { + return origNbformat; + } + + public String getTitle() { + return title; + } + + public List getAuthors() { + return authors; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Nbformat.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Nbformat.java new file mode 100644 index 0000000000..9ca414619b --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Nbformat.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public class Nbformat { + + @SerializedName("metadata") + private Metadata metadata; + + @SerializedName("nbformat") + private int nbformat; + + @SerializedName("nbformat_minor") + private int nbformatMinor; + + @SerializedName("cells") + private List cells; + + public Metadata getMetadata() { + return metadata; + } + + public int getNbformat() { + return nbformat; + } + + public int getNbformatMinor() { + return nbformatMinor; + } + + public List getCells() { + return cells; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java new file mode 100644 index 0000000000..50722d75f4 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; + +/** + * + */ +public abstract class Output { + + @SerializedName("output_type") + private String outputType; +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/RawCell.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/RawCell.java new file mode 100644 index 0000000000..c5054b8140 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/RawCell.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +/** + * + */ +public class RawCell extends Cell { + +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/RawCellMetadata.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/RawCellMetadata.java new file mode 100644 index 0000000000..1b667d7d47 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/RawCellMetadata.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; + +/** + * + */ +public class RawCellMetadata extends CellMetadata { + + @SerializedName("format") + private String format; +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java new file mode 100644 index 0000000000..a73f63550d --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public class Stream extends Output { + + @SerializedName("name") + private String name; + + @SerializedName("text") + private List text; + + public List getText() { + return text; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Note.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Note.java new file mode 100644 index 0000000000..78922b57c2 --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Note.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.zformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public class Note { + + @SerializedName("name") + private String name; + + @SerializedName("paragraphs") + private List paragraphs; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getParagraphs() { + return paragraphs; + } + + public void setParagraphs(List paragraphs) { + this.paragraphs = paragraphs; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java new file mode 100644 index 0000000000..f6a7ebfe6f --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.zformat; + +import com.google.gson.annotations.SerializedName; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * + */ +public class Paragraph { + public static final String FINISHED = "FINISHED"; + + @SerializedName("text") + private String text; + + @SerializedName("results") + private Result results; // It's a bit weird name + + @SerializedName("id") + private String id; + + @SerializedName("status") + private String status; + + public Paragraph() { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd-HHmmss"); + this.id = dateFormat.format(new Date()) + "_" + super.hashCode(); + this.status = FINISHED; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public Result getResults() { + return results; + } + + public void setResults(Result results) { + this.results = results; + } + + public String getId() { + return id; + } + + public String getStatus() { + return status; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java new file mode 100644 index 0000000000..7efa5c929d --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.zformat; + +import com.google.gson.annotations.SerializedName; +import java.util.List; + +/** + * + */ +public class Result { + public static final String SUCCESS = "SUCCESS"; + + @SerializedName("code") + private String code; + + @SerializedName("msg") + private List msg; + + public Result(String code, List msg) { + this.code = code; + this.msg = msg; + } + + public String getCode() { + return code; + } + + public List getMsg() { + return msg; + } +} diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/TypeData.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/TypeData.java new file mode 100644 index 0000000000..4aaa6423fb --- /dev/null +++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/TypeData.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.zformat; + +import com.google.gson.annotations.SerializedName; + +/** + * + */ +public class TypeData { + public static final String TABLE = "TABLE"; + public static final String HTML = "HTML"; + public static final String TEXT = "TEXT"; + + @SerializedName("type") + private String type; + + @SerializedName("data") + private String data; + + public TypeData(String type, String data) { + this.type = type; + this.data = data; + } + + public String getType() { + return type; + } + + public String getData() { + return data; + } +} diff --git a/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java b/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java new file mode 100644 index 0000000000..be9a5ec966 --- /dev/null +++ b/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.jupyter.nbformat; + +import static org.junit.Assert.assertTrue; + +import java.io.InputStream; +import java.io.InputStreamReader; +import org.apache.zeppelin.jupyter.JupyterUtil; +import org.apache.zeppelin.jupyter.zformat.Note; +import org.junit.Test; + +/** + * + */ +public class JupyterUtilTest { + + @Test + public void getNbFormat() { + InputStream resource = getClass().getResourceAsStream("/basic.ipynb"); + Nbformat nbformat = new JupyterUtil().getNbformat(new InputStreamReader(resource)); + assertTrue(nbformat.getCells().get(0) instanceof CodeCell); + + resource = getClass().getResourceAsStream("/examples.ipynb"); + nbformat = new JupyterUtil().getNbformat(new InputStreamReader(resource)); + } + + @Test + public void getNote() { + InputStream resource = getClass().getResourceAsStream("/examples.ipynb"); + Note n = new JupyterUtil().getNote(new InputStreamReader(resource), "%python", "%md"); + } + +} diff --git a/zeppelin-jupyter/src/test/resources/basic.ipynb b/zeppelin-jupyter/src/test/resources/basic.ipynb new file mode 100644 index 0000000000..472f868e63 --- /dev/null +++ b/zeppelin-jupyter/src/test/resources/basic.ipynb @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# The first two lines of the A matrix represent the coordinates of each rotor in the X,Y plane,\n", + "# and the third line the direction in which they spin.\n", + "# Note that in this example the X axis is vertical, and the Y coordinates are in the top row of A.\n", + "A = np.array([[-0.17, 0.17, -0.25, 0.25, -0.33, 0.33],\n", + " [-0.35, -0.35, 0., 0., 0.35, 0.35],\n", + " [-0.1, 0.1, 0.1, -0.1, -0.1, 0.1 ]])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Moore-Penrose pseudoinverse of A\n", + "B = np.linalg.pinv(A)\n", + "# normalize roll/pitch to the largest of both\n", + "# normalize yaw to 0.5\n", + "# and transpose\n", + "rp_max = B[:,0:1].max()\n", + "n = np.array([rp_max, rp_max, 2*B[:,2].max()])\n", + "B_nt = (B / n).T" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ -67 67 -256 256 -189 189]\n", + " [-197 -197 0 0 197 197]\n", + " [ -77 77 128 -128 -57 57]]\n" + ] + } + ], + "source": [ + "# scale and round to 256 to return final coefficients\n", + "scale = 256\n", + "coeffs = np.around(scale * B_nt).astype(int)\n", + "print(coeffs)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "# output defines\n", + "import string\n", + "rows = ['ROLL_COEF\" ', 'PITCH_COEF\" ', 'YAW_COEF\" ']\n", + "for i, r in enumerate(rows):\n", + " print('4d}'.format(c) for c in coeffs[i]], ', ') + '}\"/>')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2.0 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/zeppelin-jupyter/src/test/resources/examples.ipynb b/zeppelin-jupyter/src/test/resources/examples.ipynb new file mode 100644 index 0000000000..6213c6bef4 --- /dev/null +++ b/zeppelin-jupyter/src/test/resources/examples.ipynb @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + { + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "import numpy as np\n", + "import cvxpy as cvx\n", + "from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Nonnegative deconvolution" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEACAYAAACznAEdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFTlJREFUeJzt3X+wXGd93/H3x8gi/AiqaCupsYyx62BbmSHggozjdLqN\nXYGTie1/cByY1MbtP3UyMOkMtcQ/vprJIJxMhjLTuhOmxKiuQRFpiZUZgoUqNjMlMQ7+EXssoaqh\nFrJA1yEmBloGZPLtH/tYXq4l3V1p9+69u+/XzB2dfe5z9jzn2R+fc57zHN1UFZIknTfpBkiSlgcD\nQZIEGAiSpMZAkCQBBoIkqTEQJEnAgIGQZFuSp5I8keT+JKuTrE2yN8mhJA8mWbOg/uEkB5NsGV/z\nJUmjksXuQ0hyEfBF4PKq+mGSPwA+B2wC/qaqfjvJncDaqtqaZBNwP/B2YCOwD/jp8oYHSVrWBjlD\n+A7wQ+A1SVYBrwKOATcCO1udncBNbfkGYFdVvVBVTwOHgc2jbLQkafQWDYSq+jbwu8DX6QXB81W1\nD1hfVfOtznFgXVvlAuBo31Mca2WSpGVs0UBIcgnwm8BFwE/RO1N4L7BwCMghIUlawVYNUOdtwJeq\n6jmAJJ8Ffg6YT7K+quaTbACebfWPARf2rb+xlf2YJAaIJJ2Fqso4nneQawiHgHck+YkkAa4FDgB7\ngNtanVuBB9ryHuCWNhPpYuBS4OFTPXFV+VPFXXfdNfE2LJcf+8K+sC/O/DNOi54hVNVfJvkvwCPA\nj4DHgI8DPwnsTnI7cAS4udU/kGQ3vdA4AdxR494LSdI5G2TIiKr6HeB3FhQ/B1x3mvo7gB3n1jRJ\n0lLyTuVloNPpTLoJy4Z98RL74iX2xdJY9Ma0sW04cSRJkoaUhJrgRWVJ0gwwECRJgIEgSWoMBEkS\nYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgrQs\nfOc78N3vTroVmnWLBkKSNyV5LMmj7d/nk7w/ydoke5McSvJgkjV962xLcjjJwSRbxrsL0sr35jfD\n1VdPuhWadUP9Cc0k5wHPAFcBvwH8TVX9dpI7gbVVtTXJJuB+4O3ARmAf8NML/16mf0JTekkCq1fD\nD34w6ZZouVtOf0LzOuCvquoocCOws5XvBG5qyzcAu6rqhap6GjgMbB5BWyVJYzRsIPwK8Km2vL6q\n5gGq6jiwrpVfABztW+dYK5MkLWMDB0KS8+kd/X+mFS0c73H8R5JWsFVD1L0eeKSqvtUezydZX1Xz\nSTYAz7byY8CFfettbGUvMzc3d3K50+nQ6XSGaI4kTb9ut0u3212SbQ18UTnJp4HPV9XO9vhu4Lmq\nuvs0F5WvojdU9AW8qCydkReVNahxXlQeKBCSvBo4AlxSVd9tZa8HdtM7GzgC3FxVf9t+tw34V8AJ\n4ANVtfcUz2kgSI2BoEFNPBDGsmEDQTrJQNCgltO0U0nSlDIQJEmAgSAtG46gatIMBEkSYCBIkhoD\nQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCB\nIElqBgqEJGuSfCbJwSRPJbkqydoke5McSvJgkjV99bclOdzqbxlf8yVJozLoGcLHgM9V1RXAzwJf\nBbYC+6rqMmA/sA0gySbgZuAK4HrgniRj+YPQkqTRWTQQkrwO+KdVdS9AVb1QVc8DNwI7W7WdwE1t\n+QZgV6v3NHAY2DzqhkuSRmuQM4SLgW8luTfJo0k+nuTVwPqqmgeoquPAulb/AuBo3/rHWpmkZeT3\nfg/e/e5Jt0LLyaoB61wJ/HpVfSXJR+kNFy38k+BD/4nwubm5k8udTodOpzPsU0g6S5/8JDz00KRb\nocV0u1263e6SbCtVZ/4eT7Ie+POquqQ9/nl6gfCPgU5VzSfZAHyxqq5IshWoqrq71f88cFdVfXnB\n89Zi25ZmRQLnnw8//OHSbfPqq3uB4MdwZUlCVY3luuyiQ0ZtWOhokje1omuBp4A9wG2t7Fbggba8\nB7glyeokFwOXAg+PstGSpNEbZMgI4P3A/UnOB74GvA94BbA7ye3AEXozi6iqA0l2AweAE8AdngpI\ni/NToklbdMhobBt2yEg6KYFVq+DEiaXbpkNGK9NEh4wkSbPBQJAkAQaCJKkxECRJgIEgSWoMBGlG\n+V9OaiEDQZIEGAiSpMZAkJYJh3A0aQaCVrRrroFHHpl0K6TpYCDonH3/+3DttZPZ9p/9GezbN5lt\nj5r/hYQmzUDQOfvGN2D//km3QtK5MhAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQNCK5/x9aTQG\nCoQkTyf5yySPJXm4la1NsjfJoSQPJlnTV39bksNJDibZMq7GS5JGZ9AzhL8DOlX11qra3Mq2Avuq\n6jJgP7ANIMkm4GbgCuB64J7E/6VlmvnqStNh0EDIKereCOxsyzuBm9ryDcCuqnqhqp4GDgObkXRG\nDn1p0gYNhAK+kOQvkvzrVra+quYBquo4sK6VXwAc7Vv3WCuTJC1jqwasd01VfTPJPwT2JjlELyT6\nDX18Mzc3d3K50+nQ6XSGfQpJmmrdbpdut7sk2xooEKrqm+3fv07yR/SGgOaTrK+q+SQbgGdb9WPA\nhX2rb2xlL9MfCJKkl1t4sLx9+/axbWvRIaMkr07y2rb8GmAL8CSwB7itVbsVeKAt7wFuSbI6ycXA\npcDDI263pHPkZAAtNMgZwnrgs0mq1b+/qvYm+QqwO8ntwBF6M4uoqgNJdgMHgBPAHVVeLpOk5W7R\nQKiq/wO85RTlzwHXnWadHcCOc26dVgTjXpoO3qksSQIMBI2AY9HSdDAQpGXCYNWkGQiSJMBAkCQ1\nBoK0TDhbS5NmIEiSAANBmllexNZCBoJWPIdazo79poUMBEkSYCBoBBx6kKaDgSBJAgwESVJjIEiS\nAANBktQYCJIkYEoD4T3vgQ9/eNKtmB2Tns/uLCdpNKYyED79abj33km3Qktl0oE0Kku9HwapFprK\nQJAkDW/gQEhyXpJHk+xpj9cm2ZvkUJIHk6zpq7styeEkB5NsGUfDtXx4pClNh2HOED4AHOh7vBXY\nV1WXAfuBbQBJNgE3A1cA1wP3JH5lSNJyN1AgJNkI/CLwn/uKbwR2tuWdwE1t+QZgV1W9UFVPA4eB\nzSNprSRpbAY9Q/go8EGg/7LX+qqaB6iq48C6Vn4BcLSv3rFWJklaxlYtViHJLwHzVfV4ks4Zqg49\nR2Jubu7kcqfTodM509MPZ1pmnkiabd1ul263uyTbWjQQgGuAG5L8IvAq4CeT3AccT7K+quaTbACe\nbfWPARf2rb+xlb1MfyBIs84rbTqVhQfL27dvH9u2Fh0yqqoPVdUbquoS4BZgf1X9GvDHwG2t2q3A\nA215D3BLktVJLgYuBR4eecslSSM1yBnC6XwE2J3kduAIvZlFVNWBJLvpzUg6AdxR5QCOxsd319nx\njEQLDRUIVfWnwJ+25eeA605Tbwew45xbJ0laMt6prHPmkeZoeKajSTMQpBllAGkhA0GSBBgIkqTG\nQJAkAQaCRmDSY9Fe1JZGw0CQJAEGgqbApM9QpGlhIEgzyqE2LTS1geBR49Lxi0WaDlMbCJKk4RgI\nkiTAQJCWDYc5NWkGgiQJMBAkSc3UBoIzXyRpOFMbCI7Hzg5fa2k0pjYQJEnDMRB0zhyeW5l83bTQ\nooGQ5JVJvpzksSRPJflwK1+bZG+SQ0keTLKmb51tSQ4nOZhkyzh3QJoWfkFr0hYNhKr6AfDPq+qt\nwJuBX0hyDbAV2FdVlwH7gW0ASTYBNwNXANcD9yS+1SVpuRtoyKiq/l9bfGVb59vAjcDOVr4TuKkt\n3wDsqqoXqupp4DCweVQNliSNx0CBkOS8JI8Bx4FuVR0A1lfVPEBVHQfWteoXAEf7Vj/WyjSlJj3L\nx/PPszPp103Lz6pBKlXV3wFvTfI64MEkHWDh22not9fc3NzJ5U6nQ6fTGfYpTss3u6Rp0O126Xa7\nS7KtgQLhRVX1nSSfA94GzCdZX1XzSTYAz7Zqx4AL+1bb2Mpepj8QpFnnQYxOZeHB8vbt28e2rUFm\nGf2DF2cQJXkV8C+Ax4A9wG2t2q3AA215D3BLktVJLgYuBR4ecbulk/wilUZjkDOEfwTsbDOFzgPu\nq6r/0a4p7E5yO3CE3swiqupAkt3AAeAEcEeVH1lJWu4WDYSqehK48hTlzwHXnWadHcCOc26dVgQv\n6krTwTuVpRllkGuhqQ0E3+ySNJypDQSvWkjScKY2ECRJwzEQJEmAgSBJagwErXheL5JGw0CQJAEG\ngkbAKb6j4ZmOJm1qA8EPlyQNZ2oDQUvH8F2ZPLPTQgaCVjy/2KTRMBAkSYCBIC0bnulo0qY2EPxw\nSdJwpjYQvNA5O3ytz479poWmNhAkScMxEHTOHJ6TpoOBIEkCBgiEJBuT7E/yVJInk7y/la9NsjfJ\noSQPJlnTt862JIeTHEyyZZw7IE0Lx/Q1aYOcIbwA/Nuq+hngauDXk1wObAX2VdVlwH5gG0CSTcDN\nwBXA9cA9iYMK0nLjp1ILLRoIVXW8qh5vy98DDgIbgRuBna3aTuCmtnwDsKuqXqiqp4HDwOYRt1uS\nNGJDXUNI8kbgLcBDwPqqmodeaADrWrULgKN9qx1rZUvK029JGs6qQSsmeS3wh8AHqup7SRZ+5Q79\nFTw3N3dyudPp0Ol0hn0KSZpq3W6Xbre7JNsaKBCSrKIXBvdV1QOteD7J+qqaT7IBeLaVHwMu7Ft9\nYyt7mf5AkM6WZ4OaZgsPlrdv3z62bQ06ZPT7wIGq+lhf2R7gtrZ8K/BAX/ktSVYnuRi4FHh4BG2V\nJI3RomcISa4B3gs8meQxekNDHwLuBnYnuR04Qm9mEVV1IMlu4ABwArijymO4aeZsFWk6LBoIVfUl\n4BWn+fV1p1lnB7DjHNp1zvySWjrGvTQdvFNZK57hL43G1AaCR62SNJypDQRJZ+aZlRYyECRJgIEg\nSWoMBK14Xi+SRsNAkCQBBoJGwIuT0nSY2kBwGEE6Mz8jWmhqA0GSNBwDQZIEGAiSpGZqA8ELndKZ\n+RnRQlMbCJKk4RgIWvGcLSONxtQGgl8SkjScqQ0ESdJwDASteF4clUZj0UBI8okk80me6Ctbm2Rv\nkkNJHkyypu9325IcTnIwyZZxNVySNFqDnCHcC7xzQdlWYF9VXQbsB7YBJNkE3AxcAVwP3JN4/CZJ\nK8GigVBV/xP49oLiG4GdbXkncFNbvgHYVVUvVNXTwGFg82iaKkkap7O9hrCuquYBquo4sK6VXwAc\n7at3rJVJkpa5UV1UXnaTPJ12Kp2Zg7laaNVZrjefZH1VzSfZADzbyo8BF/bV29jKTmlubu7kcqfT\nodPpnGVzNMsMf02zbrdLt9tdkm0NGghpPy/aA9wG3A3cCjzQV35/ko/SGyq6FHj4dE/aHwhauTzS\nlMZn4cHy9u3bx7atRQMhyaeADvD3k3wduAv4CPCZJLcDR+jNLKKqDiTZDRwATgB3VE3m+M0vKUka\nzqKBUFXvOc2vrjtN/R3AjnNplCRp6XmnsiQJMBAkSc3UBoIzT6Qz8zOihaY2ECRJwzEQJEmAgaAp\n4NDH2XFqthYyECRJgIGgKeCRrjQaBoIkCZjiQHBcWZKGM7WBIEkaztQGguPKkjScqQ0ESdJwDARJ\nEmAgaAo4geDsOKyqhQwEnTO/WKTpMLWB4FGjJA1nagNBkjQcA0GSBIwxEJK8K8lXk/yvJHeOazuS\npNEYSyAkOQ/4D8A7gZ8BfjXJ5ePY1jTodruTbsKyYV+8xL54iX2xNMZ1hrAZOFxVR6rqBLALuHFM\n21rxfLO/xL54iX3xEvtiaYwrEC4AjvY9fqaVSZKWqVWT3Pgv//L4nvuZZ8b7/KN06BA88sikW3H2\nvv/93r+j6O+z6Yvf+i34ylfOfdvLQX8fjvt98cUvvnyby9VK/4ysFKkxTNhP8g5grqre1R5vBaqq\n7u6r450CknQWqmost4OOKxBeARwCrgW+CTwM/GpVHRz5xiRJIzGWIaOq+lGS3wD20rtO8QnDQJKW\nt7GcIUiSVp6J3Kk87TetJdmYZH+Sp5I8meT9rXxtkr1JDiV5MMmavnW2JTmc5GCSLX3lVyZ5ovXV\nv5/E/oxCkvOSPJpkT3s8k32RZE2Sz7R9eyrJVTPcF9taHzyR5P4kq2elL5J8Isl8kif6yka2760v\nd7V1/jzJGwZqWFUt6Q+9EPrfwEXA+cDjwOVL3Y4x7+MG4C1t+bX0rqdcDtwN/LtWfifwkba8CXiM\n3hDeG1v/vHj29mXg7W35c8A7J71/Z9knvwn8V2BPezyTfQF8EnhfW14FrJnFvmif/68Bq9vjPwBu\nnZW+AH4eeAvwRF/ZyPYd+DfAPW35V4BdA7VrAh3xDuBP+h5vBe6c9As05n3+I+A64KvA+la2Afjq\nqfoA+BPgqlbnQF/5LcB/mvT+nMX+bwS+AHR4KRBmri+A1wF/dYryWeyLtW2/17Yvuj2z9hmhF4r9\ngTCyfQc+D1zVll8B/PUgbZrEkNFM3bSW5I30jgQeovdizwNU1XFgXau2sE+OtbIL6PXPi1ZqX30U\n+CDQf8FqFvviYuBbSe5tw2cfT/JqZrAvqurbwO8CX6e3X89X1T5msC/6rBvhvp9cp6p+BPxtktcv\n1gD/t9MxSvJa4A+BD1TV9/jxL0RO8XjqJPklYL6qHgfONHd66vuC3pHwlcB/rKorgf9L7+hvFt8X\nl9AbRrwI+CngNUneywz2xRmMct8Hum9hEoFwDOi/wLGxlU2VJKvohcF9VfVAK55Psr79fgPwbCs/\nBlzYt/qLfXK68pXkGuCGJF8DPg38QpL7gOMz2BfPAEer6sX7qv8bvYCYxffF24AvVdVz7Qj2s8DP\nMZt98aJR7vvJ37X7wl5XVc8t1oBJBMJfAJcmuSjJanrjXnsm0I5x+31643sf6yvbA9zWlm8FHugr\nv6XNDLgYuBR4uJ02Pp9kc5IA/7JvnRWhqj5UVW+oqkvovdb7q+rXgD9m9vpiHjia5E2t6FrgKWbw\nfUFvosU7kvxE24drgQPMVl+EHz9yH+W+72nPAfBuYP9ALZrQxZR30XtDHAa2Tvrizhj27xrgR/Rm\nUD0GPNr2+fXAvrbve4G/17fONnqzBw4CW/rK/wnwZOurj016386xX/4ZL11Unsm+AH6W3kHR48B/\npzfLaFb74oP0AvEJYCe9WYcz0RfAp4BvAD+gdx3lffQusI9k34FXArtb+UPAGwdplzemSZIALypL\nkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRIA/x8sHdf31+NUeQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "np.random.seed(0)\n", + "n = 10000\n", + "sigma = n/10\n", + "k = 5\n", + "\n", + "x0 = np.zeros(n)\n", + "x0[np.random.choice(n,k)] = np.random.rand(k)*n/10\n", + "plt.plot(x0)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAEACAYAAACK+7BGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VOXZ//HPlyUqoohbUECUTRRRlhaxogaRtWrQ9lGw\n1qVWsdX+2tqnj9j2eYS2/irtq6211q0/W8ENedRKqigRYcQVqYAIslpFUAji1opVMVy/P+4THKdJ\nZhIyc2a53q/XvJg5ue9zrnNIcuXc25GZ4ZxzzmVLq7gDcM45V9w80TjnnMsqTzTOOeeyyhONc865\nrPJE45xzLqs80TjnnMuqjBKNpNGSVklaI+nKBspcL2mtpKWS+qerK6mjpGpJqyXNkdQhZX+HSPqn\npCuStg2UtCza13VNP13nnHO5ljbRSGoF3ACMAvoCEyT1SSkzBuhhZr2AicDNGdSdBMw1s8OBecBV\nKYf+NTA7ZdtNwEVm1hvoLWlUpifqnHMuHpnc0QwG1prZejPbDswAKlPKVALTAcxsIdBBUnmaupXA\ntOj9NGBc3c4kVQJ/B1YkbesE7GVmi6JN05PrOOecy0+ZJJrOwIakzxujbZmUaaxuuZnVAJjZZqAc\nQFJ74L+AKYBSjrExTRzOOefyTLYGAyh9kX+zI/r3auC3ZvZhC8bjnHMuJm0yKPMGcEjS5y7RttQy\nXespU9ZI3c2Sys2sJmoW2xJtPxb4iqRfAh2BWkkfAQ80cIx/I8kXcHPOuWYws+bcKDQqkzuaRUBP\nSd0klQHjgaqUMlXAeQCShgDvRc1ijdWtAi6I3p8PzAIwsxPNrLuZdQeuA/6vmd0YNa+9L2mwJEXH\nm9VQ0GbmLzOuvvrq2GPIl1eur8Uddxj7729MnWp88MFn27dvN+66y+jSxfjxj43a2uK/Fvn88mvx\n2Stb0t7RmFmtpMuBakJius3MVkqaGL5st5rZbEljJa0DtgEXNlY32vVUYKakbwDrgbMyiPcy4HZg\nd2C2mT3alJN1LlduvBF++UuYNw/69fv819q0gXPOgVNOgTPPhEsugVtvhVY+q80VqUyazoh+oR+e\nsu2WlM+XZ1o32v4OcEqa405J+fwC0K+B4s7lhZkzYepUmD8fundvuNyBB8Kjj8Lo0fCjH8G11+Yu\nRudyyf+GKnIVFRVxh5A3cnEtVqyAyy6DWbMaTzJ12reHBx+Ee+8NCSpX/PviM34tsk/ZbJeLiyQr\nxvNy+e2TT2DQILjiCrjwwqbVfeEFGDMGliyBzj5o38VEEhbTYADnXAauvRYOPRQuuKDpdQcNgm99\nCyZOBP8byRUbv6NxrgWsXg1Dh8LixdC1a/ry9fnkExg4EH7+cxjna164GGTrjsYTjXMt4Iwz4Etf\ngh/+cNf2U10d+nhWrICyspaJzblMedOZc3nq6adDH8t3vrPr+xo5Enr3hj/8Ydf35Vy+8Dsa53aB\nGZxwAnzzm83rm6nP8uVhjs3f/w7t2rXMPp3LhN/ROJeHnngCtmyBr3+95fZ51FFw/PFhEqdzxcDv\naJzbBWPGwFe+Eu5oWtKSJXDqqfDKK7D77i27b+ca4nc0zuWZpUth2bKWvZupM2BAeE2f3vL7di7X\nPNE410y/+hV873uw227Z2f/3vge//73Pq3GFzxONc81QUwOzZ8PFF2fvGMOHQ20tJBLZO4ZzueCJ\nxrlm+NOf4KtfhX32yd4xpDBk+vrrs3cM53LBBwM410S1tdCjB9x/f1g6Jps++AC6dQsrDnTrlt1j\nOeeDAZzLE48+Gpb4z3aSgbC68/jxMG1a9o/lXLZ4onGuiW65BS69NHfHu/BCuP122LEjd8d0riV5\nonGuCbZsgQUL4KxMngfbQgYNCnc2Cxbk7pjOtSRPNM41wYwZcNpp4Rd/rkjhruZPf8rdMZ1rST4Y\nwLkm+OIX4ZprwuKXufTWW9CrF7z+Ouy9d26P7UpHrIMBJI2WtErSGklXNlDmeklrJS2V1D9dXUkd\nJVVLWi1pjqQO0fYvSloSvV6UdHZSnfnRvpZIWixp/+afunNNs2oVbNwIJ5+c+2MfcACcdFJ4RLRz\nhSZtopHUCrgBGAX0BSZI6pNSZgzQw8x6AROBmzOoOwmYa2aHA/OAq6LtLwGDzGxAVO8PklonHW6C\nmQ0ws4FmtrU5J+1cc9x5J5xzDrRpE8/xzz47NN05V2gyuaMZDKw1s/Vmth2YAVSmlKkEpgOY2UKg\ng6TyNHUrgbpBm9OAcVH9j8ysbnzNHsD7ZlbbxJida1FmcNdd2VnXLFOnnw5PPQXvvBNfDM41Rya/\ntDsDG5I+b4y2ZVKmsbrlZlYDYGabgQPrCkkaLGk5sBy4IuVYt0fNZj/JIHbnWsSiRWFNs2OOiS+G\n9u1D39ADD8QXg3PNka27g+Z0Ju3svTez583sKGAg8DtJdd2f55hZP+AE4ARJ5+56qM6ld//9YckZ\ntXg3adOMHw/33htvDM41VSatzW8AhyR97hJtSy3TtZ4yZY3U3Syp3MxqJHUCtqQe2MxWS3oF6AW8\nYGabou3bJN1NaJq7s76gJ0+evPN9RUUFFRUVjZ+lcw0wg/vuC6+4jR0LF10UFvUsL487GlfoEokE\niRys2pp2eHPUEb8aGA5sAp4ndMivTCozFrjMzL4saQhwnZkNaayupKnAO2Y2NRqN1tHMJkk6FNhg\nZrWSugELgH7ANmAfM3tbUlvgbuAxM/u35xD68GbXkpYuhTPPDA8hi/uOBuBrX4OhQ+Fb34o7Elds\nYhveHHXEXw5UAyuAGVGimCjpkqjMbOBVSeuAW4BvN1Y32vVUYISkukR0bbR9KPCipMXATOASM/sH\nsBswR9JSYDGhv+ePu3oBnEsnX5rN6px5Jjz4YNxROJc5n7DpXBpHHBHWGjv22LgjCT74AA4+GDZs\ngA4d4o7GFRNfvdm5GLz8MmzbBoMHxx3JZ9q3hxNPhEceiTsS5zLjica5RjzwAJxxRv40m9UZN86b\nz1zh8ETjXCMeeggqU6cn54HTTgvPxfn447gjcS49TzTONaCmBlavDiO88k15OfTtC/Pnxx2Jc+l5\nonGuAY88AqecAmVlcUdSP28+c4XCE41zDXjoITj11LijaFhlJVRVhQmlzuUzTzTO1eOTT2DuXBgz\nJu5IGta7N7RrBy++GHckzjXOE41z9ViwIMyfOfDA9GXjNHasD3N2+c8TjXP1yPdmszpjxniicfnP\nVwZwLoVZeGzy/ffH+1iATPzrX2EE2uuvwz77xB2NK3S+MoBzObJmTZifcvTRcUeS3h57hOHXjz0W\ndyTONcwTjXMpZs8OfR/5thpAQ7yfxuU7TzTOpaiuhlGj4o4ic3X9NDt2pC/rXBw80TiX5KOP4Omn\n4eST444kcz16wN57+zBnl7880TiX5Omn4aijCq9j3UefuXzmica5JNXVMHJk3FE03ejRYZFN5/KR\nJxrnkhRqojnxRFiyJDwUzbl844nGuUhNDbz2Wn495CxT7drBF78ITzwRdyTO/TtPNM5F5s6FYcOg\nTZu4I2meU04J5+Bcvsko0UgaLWmVpDWSrmygzPWS1kpaKql/urqSOkqqlrRa0hxJHaLtX5S0JHq9\nKOnspDoDJS2L9nVd80/buX9XqM1mdUaM8ImbLj+lTTSSWgE3AKOAvsAESX1SyowBephZL2AicHMG\ndScBc83scGAecFW0/SVgkJkNiOr9QVLr6Gs3AReZWW+gt6QCmu3g8plZ4SeagQPhzTdh06a4I3Hu\n8zK5oxkMrDWz9Wa2HZgBpD7cthKYDmBmC4EOksrT1K0EpkXvpwHjovofmVnd1LM9gPfNrFZSJ2Av\nM1sUfW16XR3ndtXy5bDnntC9e9yRNF/r1mH+jzefuXyTSaLpDGxI+rwx2pZJmcbqlptZDYCZbQZ2\nLsguabCk5cBy4IqkY2xME4dzzfLYY4V9N1PH+2lcPspWt2dzVonaudyymT0PHBU1sz0qqclPRp88\nefLO9xUVFVRUVDQjJFcqqqvh0kvjjmLXjRgBP/1paAoslLXaXHwSiQSJRCLrx8kk0bwBHJL0uUu0\nLbVM13rKlDVSd7OkcjOriZrFtqQe2MxWSXoF6NXIMeqVnGica8zHH4cVAWbMiDuSXde9O+y2G6xc\nCUceGXc0Lt+l/hE+ZcqUrBwnk6azRUBPSd0klQHjgaqUMlXAeQCShgDvRc1ijdWtAi6I3p8PzIrq\nH1rX+S+pG9CT0M+zGXg/alZTdLxZzThn5z7n+eehT5/CW3amPpKPPnP5J22iMbNa4HKgGlgBzDCz\nlZImSrokKjMbeFXSOuAW4NuN1Y12PRUYIWk1MBy4Nto+FHhR0mJgJnCJmf0j+tplwG3AGkLy8UU3\n3C6bN6+wFtFMx/tpXL7xJ2y6kldRAZMmhfXCisHbb4cmtK1boW3buKNxhcSfsOlcFvzrX/C3v4Wn\nVBaL/fYLiWbRovRlncsFTzSupD3zDBxzDLRvH3ckLWvYMJjf5LGazmWHJxpX0ubNC7+Ui40nGpdP\nPNG4kjZ/fnENBKhz4omwcGEYuu1c3DzRuJL1z3/CsmVw3HFxR9LyOnQIQ7YXLow7Euc80bgS9tRT\n4Rkue+wRdyTZ4c1nLl94onElq1j7Z+p4onH5whONK1nz5xd3ohk6NAzd/uijuCNxpc4TjStJ774L\nq1fDscfGHUn27LUX9OsHzz4bdySu1HmicSVpwYIwCKCsLO5Issubz1w+8ETjSlKxrW/WEE80Lh94\nonElqdj7Z+ocfzwsWQIffhh3JK6UeaJxJeett2D9ehg0KO5Isq9dOxgwIDxvx7m4eKJxJSeRgBNO\ngDbZer5snvHmMxc3TzSu5BT7/JlUnmhc3DzRuJJTKv0zdY47DpYvD0vuOBcHTzSupGzaBFu2QP/+\ncUeSO7vvDl/4Qlhyx7k4eKJxJeWJJ0L/TKsS+86vqPDmMxefEvtxc6UukQi/dEuN99O4OGWUaCSN\nlrRK0hpJVzZQ5npJayUtldQ/XV1JHSVVS1otaY6kDtH2UyT9TdKLkhZJGpZUZ360ryWSFkvav/mn\n7kpRqSaaY4+FVavg/ffjjsSVorSJRlIr4AZgFNAXmCCpT0qZMUAPM+sFTARuzqDuJGCumR0OzAOu\nira/BZxqZscAFwB3pIQ0wcwGmNlAM9vaxPN1JWzTJqipgaOPjjuS3NttNxg8GJ58Mu5IXCnK5I5m\nMLDWzNab2XZgBlCZUqYSmA5gZguBDpLK09StBKZF76cB46L6L5rZ5uj9CmB3SW2bGLNz/2bBgvDk\nydat444kHsOGhTs653Itk1/anYENSZ83RtsyKdNY3XIzqwGIEsuBqQeW9FVgcZSk6tweNZv9JIPY\nndspkYCTToo7ivj4gAAXl2zNjVYz6tjndiD1BX4BjEjafI6ZbZK0J/CApHPN7M76djZ58uSd7ysq\nKqgoxYZ59zmJBFx8cdxRxGfwYFizBt57D/bZJ+5oXD5IJBIkcnCbm0mieQM4JOlzl2hbapmu9ZQp\na6TuZknlZlYjqROwpa6QpC7AA8DXzey1uu1mtin6d5ukuwlNc2kTjXObN4fXMcfEHUl8yspgyJDQ\nhHj66XFH4/JB6h/hU6ZMycpxMmk6WwT0lNRNUhkwHqhKKVMFnAcgaQjwXtQs1ljdKkJnP8D5wKyo\n/j7AQ8CVZvZc3QEktZa0X/S+LXAqsLxpp+tKVd38mVLtn6nj/TQuDmkTjZnVApcD1cAKYIaZrZQ0\nUdIlUZnZwKuS1gG3AN9urG6066nACEmrgeHAtdH2y4AewP+kDGPeDZgjaSmwmNDf88ddvgKuJDzx\nRGkOa07l/TQuDjKz9KUKjCQrxvNyzXfkkXDHHaXxaIDGbN8O++0Hr70G++4bdzQu30jCzJrTx94o\nHyrsil5NDbz5Zmmtb9aQtm3DIpsLFsQdiSslnmhc0fP+mc/zfhqXa55oXNEr1WVnGuL9NC7XPNG4\noucDAT5v0CB49VV4++24I3GlwhONK2pbtsAbb3j/TLK2beH440MCdi4XPNG4ovbEEzB0qPfPpPJ+\nGpdLnmhcUfP+mfp5P43LJU80rqh5oqnfwIHw+uvw1ltxR+JKgScaV7S8f6ZhbdqEJkXvp3G54InG\nFa0FC0Knd5tsrVFe4LyfxuWKJxpXtLzZrHHeT+NyxRONK1qeaBo3YEBoWtyyJX1Z53aFJxpXlN56\nCzZsCL9MXf1atw5L83jzmcs2TzSuKC1YEDq7vX+mcd5P43LBE40rSokEnHRS3FHkP++ncbngicYV\nJe+fycwxx3z2mGvnssUTjSs6b70VJiMOHBh3JPmvdWs48URvPnPZ5YnGFR2fP9M03k/jss0TjSs6\n/liApvF+GpdtGSUaSaMlrZK0RtKVDZS5XtJaSUsl9U9XV1JHSdWSVkuaI6lDtP0USX+T9KKkRZKG\nJdUZKGlZtK/rmn/arpjNn+8DAZri6KNh69bwuGvnsiFtopHUCrgBGAX0BSZI6pNSZgzQw8x6AROB\nmzOoOwmYa2aHA/OAq6LtbwGnmtkxwAXAHUmHugm4yMx6A70ljWryGbuiVlMT5s8MGhR3JIWjVSvv\np3HZlckdzWBgrZmtN7PtwAygMqVMJTAdwMwWAh0klaepWwlMi95PA8ZF9V80s83R+xXA7pLaSuoE\n7GVmi6I60+vqOFdn/vzwS9P7Z5qmosITjcueTBJNZ2BD0ueN0bZMyjRWt9zMagCixHJg6oElfRVY\nHCWpzlH9xuJwJW7ePBg+PO4oCs+wYd5P47InW3/3qRl17HM7kPoCvwBGNCeAyZMn73xfUVFBhfcO\nl4R58+A734k7isJz1FHw7ruwcSN06RJ3NC5XEokEiRzcymaSaN4ADkn63CXallqmaz1lyhqpu1lS\nuZnVRM1iO5f2k9QFeAD4upm9luYY9UpONK40rF8P//gH9O0bdySFp1WrMIAikYBzz407GpcrqX+E\nT5kyJSvHyaTpbBHQU1I3SWXAeKAqpUwVcB6ApCHAe1GzWGN1qwid/QDnA7Oi+vsADwFXmtlzdQeI\nmtfelzRYkqLjzWri+boiNn9+aAJq5YP2m8X7aVy2pP2RNLNa4HKgGlgBzDCzlZImSrokKjMbeFXS\nOuAW4NuN1Y12PRUYIWk1MBy4Ntp+GdAD+B9JSyQtlrR/0tduA9YQBhk8umun74qJ98/sGu+ncdki\nM0tfqsBIsmI8L9cws9C38MQT0LNn3NEUph07oLwcXngBDjkkfXlXfCRhZs3pY2+UNzK4orBmTVi3\nq0ePuCMpXK1aefOZyw5PNK4ozJsHJ58MavG/xUqLJxqXDZ5oXFGoSzRu13g/jcsGTzSu4O3YEX45\neqLZdUccAR9+CK+9Fnckrph4onEF76WXYL/9fKJhS5C8+cy1PE80ruB5s1nLGjYsXFPnWoonGlfw\nHn/cE01LGjECHnssDBl3riV4onEFbft2ePJJf9BZS+rRA9q1g+XL447EFQtPNK6gPf98+MV4wAFx\nR1Jc6u5qnGsJnmhcQauuhpEj446i+Iwc6YnGtRxPNK6geaLJjpNPhqeego8+ijsSVww80biC9e67\nsGIFHH983JEUn332CY9beOaZuCNxxcATjStY8+bB0KGw225xR1KcRo4Md4zO7SpPNK5gebNZdvmA\nANdS/DEBriCZwWGHwezZcOSRcUdTnLZvh/33h3XrfFRfqfDHBDiXZN06+PTTsDaXy462bcPjnR9/\nPO5IXKHzROMKUl2zmT8WILu8+cy1BE80riB5/0xu1M2n8ZZotys80biCs317WF34lFPijqT49e4d\n/l29Ot44XGHLKNFIGi1plaQ1kq5soMz1ktZKWiqpf7q6kjpKqpa0WtIcSR2i7ftKmifpn5KuTznG\n/GhfSyQtlrR/807bFbLnnoNevUJHtcsuyYc5u12XNtFIagXcAIwC+gITJPVJKTMG6GFmvYCJwM0Z\n1J0EzDWzw4F5wFXR9o+AnwA/aCCkCWY2wMwGmtnWjM/UFY05c0LfgcuNMWPgkUfijsIVskzuaAYD\na81svZltB2YAlSllKoHpAGa2EOggqTxN3UpgWvR+GjAuqv+hmT0DfLwLMbsiNns2jB0bdxSl45RT\n4OmnYdu2uCNxhSqTX9qdgQ1JnzdG2zIp01jdcjOrATCzzcCBGcZ8e9Rs9pMMy7si8uab4THDxx0X\ndySlo0MH+MIXwuOynWuONlnab3MGnWYyruUcM9skaU/gAUnnmtmd9RWcPHnyzvcVFRVU+ANLisIj\nj4Q+gzbZ+s519Ro7NtxJnnpq3JG4lpRIJEjk4Lndmfy4vgEckvS5S7QttUzXesqUNVJ3s6RyM6uR\n1AnYki4QM9sU/btN0t2Eprm0icYVj4cfhjPOiDuK0jN2bOirMfO5S8Uk9Y/wKVOmZOU4mTSdLQJ6\nSuomqQwYD1SllKkCzgOQNAR4L2oWa6xuFXBB9P58YFY9x975LS2ptaT9ovdtgVMBfwZgCfnkk7CQ\n5ujRcUdSeo44IiSYl1+OOxJXiNLe0ZhZraTLgWpCYrrNzFZKmhi+bLea2WxJYyWtA7YBFzZWN9r1\nVGCmpG8A64Gz6o4p6VVgL6BMUiUwEngdmCOpDdAamAv8sQWugSsQTz4Jffr4ultxkODLXw7NZ337\nxh2NKzS+qKYrGFdcAR07wn//d9yRlKaHH4Zf/SpMlnXFKVuLanqicQWjTx+46y4YNCjuSErThx9C\np06wYUMYieaKj6/e7EraK6/A++/DgAFxR1K62rULD5rzRTZdU3micQXh4YfDqKdW/h0bq7Fjw/+F\nc03hP7auIDz8cOiMdvGqGxBQWxt3JK6QeKJxee+99+DZZ2HUqLgjcYcdFvppnnsu7khcIfFE4/Le\nI4+EJz22bx93JA5g3Dj4y1/ijsIVEk80Lu89+GD45ebyw7hx4f/EB3a6THmicXnt44/DYwFOOy3u\nSFyd/v3h009hxYq4I3GFwhONy2vz5kG/fnBgpmt7u6yTPrurcS4TnmhcXvvLX3wRzXzkicY1ha8M\n4PJWbS107hweutWjR9zRuGSffgoHHQQvvACHHJK+vCsMvjKAKzkLF4YmM08y+adNm/Bsmln1rbnu\nXApPNC5v+Wiz/ObDnF2mvOnM5SUz6N49/CLr3z/uaFx9PvwQDj4Y1qzxwRrFwpvOXElZtAjKyuCY\nY+KOxDWkXbuw/tz998cdict3nmhcXrr3Xjj7bH9scL47++zwf+VcY7zpzOWdHTugWzd49FF/mmO+\n++ijMPpsxYrQjOYKmzeduZLx7LPhwVqeZPLf7ruHVRvuuy/uSFw+80Tj8s7MmXDWWXFH4TLlzWcu\nnYwSjaTRklZJWiPpygbKXC9praSlkvqnqyupo6RqSaslzZHUIdq+r6R5kv4p6fqUYwyUtCza13XN\nO2WXz2pr4X//N/zycoVhxAhYtSo84tm5+qRNNJJaATcAo4C+wARJfVLKjAF6mFkvYCJwcwZ1JwFz\nzexwYB5wVbT9I+AnwA/qCecm4CIz6w30luRPKCkyTz0VhsoefnjckbhMlZWFOTUzZ8YdictXmdzR\nDAbWmtl6M9sOzAAqU8pUAtMBzGwh0EFSeZq6lcC06P00YFxU/0Mzewb4OPkAkjoBe5nZomjT9Lo6\nrnjcfTeMHx93FK6pxo+HGTPijsLlq0wSTWcg+aZ4Y7QtkzKN1S03sxoAM9sMpJvy1Tmq31gcroD9\n61+h2ezcc+OOxDXVySfDm2/CypVxR+LyUZss7bc5w+NadDzy5MmTd76vqKigoqKiJXfvsuCvf4VB\ng6BLl7gjcU3VujV87WswbRpce23c0bhMJRIJEolE1o+TSaJ5A0hen7VLtC21TNd6ypQ1UnezpHIz\nq4maxbZkEEd9x6hXcqJxhWH6dDjvvLijcM11/vkwahRcc01IPC7/pf4RPmXKlKwcJ5Oms0VAT0nd\nJJUB44GqlDJVwHkAkoYA70XNYo3VrQIuiN6fD9S3DuzOO6Ooee19SYMlKTqerx1bJGpqwuMA/Nkz\nhatvX+jUCR5/PO5IXL5Je0djZrWSLgeqCYnpNjNbKWli+LLdamazJY2VtA7YBlzYWN1o11OBmZK+\nAawHds6ckPQqsBdQJqkSGGlmq4DLgNuB3YHZZvZoC1wDlwfuuQcqK6F9+7gjcbvi/PND89nIkXFH\n4vKJL0Hj8sKAAfDrX4dOZVe4tm6Fnj3h9ddh773jjsY1lS9B44rW0qXwzjvg4zUK3/77w7BhPqfG\nfZ4nGhe7W26Bb34TWvl3Y1G46CL44x/jjsLlE286c7H64IPwzPnly33132JRWwuHHRaekDpwYNzR\nuKbwpjNXlO65B046yZNMMWndGi65JNypOgd+R+NiNmhQmHcxenTckbiWtGkTHHkkrF/vgwIKid/R\nuKLzt7+FQQA+FLb4HHQQDB8Od90VdyQuH3iicbG5+Wa4+GIfBFCsLr0UbroJvHHBedOZi8Vbb0Hv\n3rBmDRxwQNzRuGzYsQP69IHbboMTTog7GpcJbzpzReXmm+GrX/UkU8xatYLvfhd++9u4I3Fx8zsa\nl3MffwyHHgpz54b1sVzx2rYt/F8/+2xYMcDlN7+jcUXjnnvg6KM9yZSCPfcMQ52v8wevlzS/o3E5\nZQb9+8MvfxmWlHfFb9Om8EfFunWw775xR+Ma43c0rihUV4eZ4z6kuXQcdBCcfrpP4CxlfkfjcsYs\njD667DKYMCHuaFwuLV8OI0bAK69Au3ZxR+Ma4nc0ruAlErBlC5x1VtqirsgcdRQcf3wYbehKj9/R\nuJw5+eTwqOYLLog7EheHZctCv5zf1eQvv6NxBe3pp+HVV+FrX4s7EheXo4+G446DW2+NOxKXa35H\n47LOLNzNnHNOWHLGla6lS2Hs2HBXs8cecUfjUvkdjStYc+bAm2/ChRfGHYmLW//+4a7m97+POxKX\nSxklGkmjJa2StEbSlQ2UuV7SWklLJfVPV1dSR0nVklZLmiOpQ9LXror2tVLSyKTt86N9LZG0WNL+\nzTttlys7dsCkSfCLX0CbNnFH4/LBL34R5lFt3Rp3JC5X0iYaSa2AG4BRQF9ggqQ+KWXGAD3MrBcw\nEbg5g7qzQjthAAAMJklEQVSTgLlmdjgwD7gqqnMkcBZwBDAGuFFS8q3cBDMbYGYDzcy/VfPcPffA\n7rvDGWfEHYnLF717w/jx8LOfxR2Jy5VM7mgGA2vNbL2ZbQdmAJUpZSqB6QBmthDoIKk8Td1KYFr0\nfhowLnp/OjDDzD41s9eAtdF+mhKzywMffgg//jFMnQpq8VZfV8iuvjo8q2bt2rgjcbmQyS/tzsCG\npM8bo22ZlGmsbrmZ1QCY2WbgwAb29UbK8W6Pms1+kkHsLkbXXgvHHhse1excsgMOgB/8AP7zP+OO\nxOVCtlrNm/P3aybDxM4xs02S9gQekHSumd1ZX8HJkyfvfF9RUUFFRUUzQnLNtW4d3HhjGGXkXH2u\nuCIMea6qCkvUuNxLJBIkEomsHyeTRPMGcEjS5y7RttQyXespU9ZI3c2Sys2sRlInYEuafWFmm6J/\nt0m6m9CkljbRuNwyC88h+eEPoUuXuKNx+Wq33cIfIxddFB77vOeecUdUelL/CJ8yZUpWjpNJ09ki\noKekbpLKgPFAVUqZKuA8AElDgPeiZrHG6lYBF0TvzwdmJW0fL6lM0mFAT+B5Sa0l7Rcdoy1wKrC8\nqSfssu++++Dvf4fvfz/uSFy+Gz48LE3jAwOKW0YTNiWNBn5HSEy3mdm1kiYCZma3RmVuAEYD24AL\nzWxxQ3Wj7fsCMwl3L+uBs8zsvehrVwEXAduB75pZtaR2wALCXVhrYC5wRX0zM33CZny2bAnNIQ8+\nCEOGxB2NKwSbN4fvmdmz4QtfiDua0patCZu+MoBrMWbwH/8B3buHeRLOZeruu8NdzeLFvmJAnDzR\nNIEnmnjccw/89KewZEmYO+NcpszC3JqDD4bf/jbuaEqXJ5om8ESTe+vWhaVFHn0UBg2KOxpXiN55\nJzSh/fnP4dk1Lvd8rTOXtz76KDxj5uqrPcm45tt3X7jjjvAoiQ0b0pd3hcPvaNwuu/RSePttmDnT\nVwBwu+6Xv4T774cFC8IQaJc73nTWBJ5ocuf3vw9PTXzmGejQIX1559Ixg698BfbbLzy7xv94yR1v\nOnN5Z/bssBLvQw95knEtR4Jp02DRorCMkSt8vnC7a5ZFi8IjmWfNgsMOizsaV2z22iv8IfOlL4XV\nJb7+9bgjcrvCE41rshdfhFNPhdtuCyPNnMuGgw8OyWbYMGjf3h81Ucg80bgmWbECRo+GG26A006L\nOxpX7I48Eh55BMaMCX03Z54Zd0SuOTzRuIw980z4Qf/Nb8IKAM7lwsCBYX7WmDFhKP0558QdkWsq\nTzQuI3/9K3zjG2Gew+jRcUfjSs2AATB3Lnz5y/Dqq/CjH/lotELiw5tdo3bsgGuuCUOYH3ggPMjM\nubhs2hSabPv2hZtugnbt4o6ouPjwZpdzb78N48aFZotFizzJuPgddBA88QR8+ikMHgwvvxx3RC4T\nnmhcvaqqoF8/6NUL5s8PI4Ccywd77gl33hmed3TiiWHScG1t3FG5xnjTmfucDRvgyith4UK4/XY4\n4YS4I3KuYatWwSWXwCefhFUEjj467ogKmzeduaz65z9h8mTo3x969IBlyzzJuPzXpw8kEuFx0Kec\nEgasvP563FG5VJ5oStzbb4dVl7t3h7Vrw4OnfvYzf367KxytWsHFF8OaNaEPZ8AAuOyy8NnlB080\nJcgMnnoqLCHTsye8+WaYI3PXXdCtW9zROdc8++wTRkiuWAEdO8LQoWGEWlVVaFpz8cko0UgaLWmV\npDWSrmygzPWS1kpaKql/urqSOkqqlrRa0hxJHZK+dlW0r5WSRiZtHyhpWbSv65p3yqVpxw547jmY\nNAkOPzz8BdivH6xeDX/8Y+j0d64YdOoEP/85vPYaVFbCr34FnTuHu5zHHoOPP447wtKTNtFIagXc\nAIwC+gITJPVJKTMG6GFmvYCJwM0Z1J0EzDWzw4F5wFVRnSOBs4AjgDHAjdLOqVk3AReZWW+gt6RR\nzT3xYrdjByxfDt//foIJE8IP2kUXQevW4c7l5ZfhBz+AAw+MO9LcSSQScYeQN0rhWrRrB9/8Jjz5\nJDz/fPgZuPpqOOCAcKfzm9/As89CdXUi7lCLXiYrAwwG1prZegBJM4BKYFVSmUpgOoCZLZTUQVI5\ncFgjdSuBk6L604AEIfmcDswws0+B1yStBQZLWg/sZWaLojrTgXHAnOaceLHYvj2MFFu/HlauDJ34\nL70UXgccAHvtleC7363gmmtCP0wpSyQSVFRUxB1GXii1a3HYYWE1gR/9KPRLPvZYSEB33gkvvZRg\nwIAK+vYNa6sdcUQYZNC1qz94raVkkmg6A8kPVt1ISD7pynROU7fczGoAzGyzpLq/rTsDzybVeSPa\n9mlUP/UYRW3BgjCKZuvW8AOydWt4bd4cmga2bAkdoN26hSaxfv1gwoTw7777hpFkF14Y91k4lz/2\n2w/Gjw8vCMln7Nhwl79yJTz+eGhSfvPN0O/TtWt4VMHBB4efqf32C69994Xhw2GPPeI9n0KQrbXO\nmjMOu0UnvhTDysI1NWFG/oQJsP/+4dWvX/i3vDwkl86doW3buCN1rnCVlYWBA0OHfn77jh3hZ3DD\nBti4MSx/8/bbofVg8eIwrLq2FgYNiiXsFldWlsWdm1mjL2AI8GjS50nAlSllbgbOTvq8CihvrC6w\nknBXA9AJWFnf/oFHgWOTy0TbxwM3NRCz+ctf/vKXv5r+SpcTmvPK5I5mEdBTUjdgE+EX/ISUMlXA\nZcC9koYA75lZjaStjdStAi4ApgLnA7OStt8l6beEprGewPNmZpLelzQ4iuk84Pr6As7GzFbnnHPN\nkzbRmFmtpMuBasIotdvMbKWkieHLdquZzZY0VtI6YBtwYWN1o11PBWZK+gawnjDSDDN7WdJM4GVg\nO/DtpPVkLgNuB3YHZpvZoy1wDZxzzmVRUa515pxzLn8U1coAmUwsLXSSukiaJ2mFpJck/Z9oe0lO\ngJXUStJiSVXR55K8DgDRtIL/jc5vhaRjS/F6ROe1IjqHuySVldJ1kHSbpBpJy5K2tdj5R9dzRlTn\nWUmHpA0qGx0/cbwISXMd0A1oCywF+sQdVxbOsxPQP3rfHlgN9CE0Rf5XtP1K4Nro/ZHAEkIz6aHR\nNaq7k10IfDF6PxsYFff5NeN6fB+4E6iKPpfkdYhivx24MHrfBuhQatcj+vn/O1AWfb6X0AdcMtcB\nGAr0B5YlbWux8we+BdwYvT+bMO+x0ZiK6Y5m58RSM9sO1E0OLSpmttnMlkbvPyCM3utCONdpUbFp\nhMmskDQB1sxeA+omwHai/gmwBUNSF2As8P+SNpfcdQCQtDdwgpn9GSA6z/cpvevxD+ATYE9JbYA9\nCHPxSuY6mNlTwLspm1vy/JP3dR8wPF1MxZRoGpo0WrQkHUr4y+U5UibAAskTYJOvS90E2M4U/gTY\n3wI/JAzLrFOK1wHCKhxbJf05akq8VVI7Sux6mNm7wK+B1wnn9L6ZzaXErkM9DmzB899Zx8xqgfck\n7dvYwYsp0ZQUSe0Jf018N7qzSR3VUdSjPCR9GaiJ7u4aG85e1NchSRtgIPAHMxtIGP05idL7vuhO\naE7tBhxMuLP5GiV2HTLQkuefdjpJMSWaN4DkTqku0baiEzUJ3AfcYWZ1849qFNaXI7rt3RJtfwPo\nmlS97ro0tL1QHA+cLunvwD3AyZLuADaX2HWosxHYYGZ/iz7fT0g8pfZ98QXgaTN7J/pr+y/Alyi9\n65CqJc9/59cktQb2NrN3Gjt4MSWanRNLJZURJodWxRxTtvwJeNnMfpe0rW4CLPz7BNjx0UiRw/hs\nAuxm4H1JgyWJMAF2FgXCzH5kZoeYWXfC//U8M/s68FdK6DrUiZpFNkjqHW0aDqygxL4vCINjhkja\nPYp/OGFOXqldB/H5O42WPP+qaB8A/0FYfb9xcY+QaOHRFqMJ32hrgUlxx5OlczweqCWMqlsCLI7O\ne19gbnT+1cA+SXWuIowmWQmMTNo+CHgpul6/i/vcduGanMRno85K+TocQ/iDaynwAGHUWcldD0K/\n3QpgGaHTum0pXQfgbuBN4GNCX9WFQMeWOn9gN2BmtP054NB0MfmETeecc1lVTE1nzjnn8pAnGuec\nc1nlicY551xWeaJxzjmXVZ5onHPOZZUnGuecc1nlicY551xWeaJxzjmXVf8fRf4Kw8z+MEMAAAAA\nSUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "c = np.exp(-np.arange(-n/2., n/2.)**2./(2*sigma**2))/np.sqrt(2*sigma**2*np.pi)\n", + "c[c < 1e-6] = 0\n", + "plt.plot(c)" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 65, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEACAYAAAC3adEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmUFNXZBvDnHWBYBFSCILKpQQFRIKAIQXCUo+KKEY8f\nuAWNW/zALYngEpkgBvcVNaBicAM/N8AdNY6oBERFQWQZBASG1QUEkRlm5v3+uF1WVW/TM11dVd39\n/M7pU7eWrrpT011v17117xVVBRERkaUg6AwQEVG4MDAQEZELAwMREbkwMBARkQsDAxERuTAwEBGR\niyeBQUQGi8gyEVkhIqMTbFMkIgtF5CsRed+L4xIRkfck3XYMIlIAYAWAQQA2AFgAYJiqLnNsszeA\nuQBOVNUyEWmpqt+ldWAiIsoIL+4Y+gAoVdVvVXUPgOkAhkRtcy6Al1S1DAAYFIiIwsuLwNAWwDrH\n/PrIMqdDAbQQkfdFZIGIXODBcYmIKAPq+3icXgCOB7AXgP+KyH9VdaVPxyciohR5ERjKAHRwzLeL\nLHNaD+A7Vd0NYLeIzAHQA0BMYBARdt5ERFRLqipe7cuLoqQFADqJSEcRKQQwDMCsqG1mAjhGROqJ\nSBMARwNYmmiHqsqXB6+xY8cGnodcevF88nyG9eW1tO8YVLVKREYCmA0TaJ5Q1aUicrlZrZNVdZmI\nvA1gEYAqAJNV9et0j01ERN7zpI5BVd8C0Dlq2aSo+bsB3O3F8YiIKHPY8jmHFRUVBZ2FnJIN5/PT\nT4F77gk6F6nJhvOZr9Ju4OY1EdGw5YkoW5x9NvDSSwC/QvlFRKAhq3wmopAoLAw6B5QLGBiIsty2\nbcDy5cCECcCMGfby8eN550B1w6Ikoix37rnAtGnuZVVVQL16wK5dQOPGweSL/MOiJCJy2bkzdtme\nPWZaVASsWeNnbigXMDAQZanqamD1auD112PXNWpkpp98Asyc6W++KPsxMBBlqTffBA4+2ASIZLZu\n9Sc/lDsYGIiyVMOGqW13222AeFb6TPmAgYEoSzVoULvt33svM/mg3MPAQJSlrArmVN13X2byQbmH\ngYEoS51wQu2251PglCoGBqIs9HVU38S9eiXetksXM62pkprIwsBAlIVWrHDPd+0KvPOOeQFAt272\nuocfNlPeMVCq/Brak4g8sngxMHmye9l99wH77WfP9+4NLFli0oceaqYMDJQq3jEQZZnu3U0bBouq\nOyj85S/ApZeaxm+A3dituho47jhg1Sr/8krZiX0lEWWRTz8FjjrKvSzR16WqCqhfH/jpJ6B5c3v5\n888D55yTuTyS/7zuK4mBgSiLnHMO8MIL9nyjRsAvv8TfVhUoKADKy2Mbw/Erllu8DgysYyDKIvvs\n455/+eXE24qYAMAgQLXFOgaiLFI/6qfcySfX/B4R06eS0x13AH/7m3f5otzCoiSiLDFtmhl7wSnV\nr8r69UD79rHL+VXLDaxjIMpT8TrCS/WrUl5uP51Ul/dTuLGOgYjw7LPAQQelvn2qPbESAR7VMYjI\nYBFZJiIrRGR0nPXHisg2Efk88rrZi+MS5YvPP7fTQ4eaIqV+/Wq3D1WgVStv80W5Ke3AICIFACYC\nOAlANwDDRaRLnE3nqGqvyGt8usclygcjRpi2CyUl9rJ0fv2Xl7vn+/e3W0gTWby4Y+gDoFRVv1XV\nPQCmAxgSZzsOFUJUS1Onmkpn5wU9Xl1Bql56CRjv+Fk2d6476BAB3gSGtgDWOebXR5ZF6yciX4jI\n6yJymAfHJcoLv/wC3HijPd+pU933NWhQbKtnju5G0fyqfP4MQAdV3SUiJwOYAeDQRBsXFxf/mi4q\nKkJRUVGm80cUWmVlZjp0qOk8b++909vfgQe65xkYsk9JSQlKMnirl/bjqiLSF0Cxqg6OzI8BoKp6\nR5L3rAbQW1V/iLOOj6sSRTgv2uedBzzzjDf7vfde09keADz6KHDFFd7sl4Lh9eOqXhQlLQDQSUQ6\nikghgGEAZjk3EJHWjnQfmIAUExSIyPbdd+75dOoWoo0aZac5gA9FSzswqGoVgJEAZgNYAmC6qi4V\nkctF5LLIZmeLyFcishDA/QD+J93jEuW6Dh3c802aeLfvBg2AkSNNuqLCu/1SbmDLZ6KQii77//57\noEULb4/Rty8wfz5bQGe7MBYlEVGGXXyx90EBALZu9X6flP14x0AUQlu3ulspZ+or0aEDsG4d7xiy\nHe8YiPLAlCn+HKey0p/jUHZhYCAKoehxFzJlzx4z5YA+5MTAQBRC9er5c5xx48y0oMBu10DEwEAU\nQlVV/hzH2bBt7lx/jknhx8BAFEKvvGKnr7kmc8dxPhLLhm5kYWAgCpkPPgA+/tiev+8+f47r110K\nhR8DA1HIDB/u7/GscaQZGMjCwEAUMhs32ul//jPzx2vc2EyjB/Gh/MXAQBQi8+bZ6e7dgRtuyPwx\n99nHTHftyvyxKDuw5TNRSFRUAO3bA1u2mPkffgD23Tfzx925E2jWzKT51ctObPlMlKNWrrSDAuBP\nUACApk3ttNXgjfIbAwNRSITh1/q2bUHngMKAgYEoJH4IwdBVrGcggIGBKDQGDgw6B8DixUHngMKA\nlc9EIRE9MI+fXwPnsbdvB5o39+/YlD5WPhPloA8+cM+/+qq/x3d2u8EKaGJgIAqBoiL3/Gmn+Xt8\n5/jSDAzEwEAUsOjurq0GZ366+mpg0SKTXr3a/+NTuLCOgShgnTsDK1bY87t22d1U+E0EaN0a2LQp\nmONT3bCOgSjHFDi+hY0bBxcULLt3B3t8Ch4DA1GANm4Eli2z53/3u+DyYtm+PegcUNA8CQwiMlhE\nlonIChEZnWS7o0Rkj4ic5cVxibLdo4+6559+Oph8RGNDt/yWdmAQkQIAEwGcBKAbgOEi0iXBdrcD\neDvdYxLlCmexUYsWwMEHB5cXp507g84BBcmLO4Y+AEpV9VtV3QNgOoAhcbYbBeBFAFvirCPKS87A\n0LZtcPmI9tNPQeeAguRFYGgLYJ1jfn1k2a9E5AAAZ6rqowA8qzknynYtW9rpMDyM16CBmVqjulF+\nqu/Tce4H4Kx7SBociouLf00XFRWhKLr1D1GOuPdeO92kSXD5sFRUmEdWV64MOieUTElJCUpKSjK2\n/7TbMYhIXwDFqjo4Mj8GgKrqHY5tVllJAC0B/AzgMlWdFWd/bMdAOe+UU4C77gIOP9zMn3028NBD\nwP77B5svwASG5s35dFI28bodgxd3DAsAdBKRjgA2AhgGwDWcuar+WqUmIk8CeDVeUCDKF2++CZx+\nuj3fpUs4goKFdQz5Le06BlWtAjASwGwASwBMV9WlInK5iFwW7y3pHpMoF5SV2enonlXD4Ouvg84B\nBcWTOgZVfQtA56hlkxJse7EXxyTKdrfdZqfDGBjWrwcOOyzoXFAQ2PKZKATCGBhefRUoLw86FxQE\nBgYin8V7tiJMgWHffc104kTgwQeDzQsFg4GByGdTp9ppZzuGsHAOErRjR3D5oOAwMBD57KKL7LTV\ndqEgRN/E3/8euDhSE8hBe/JTiD6ORPmnd28zDVNRkggwcKBJMzDkJwYGIh999517/vnnzbSw0P+8\nJGN1jbFhQ7D5oGAwMBD5aPly93yDBsC8ecCoUcHkJ5HNm8102rRg80HB8KuvJCICUF0du+zoo/3P\nR02cdzYVFeG7o6HM4h0DkY+eeCLoHKTGWRk+OuHQW5SrGBiIfFBaCvzwg7sbjDBr0cJOr10bXD4o\nGGn3ruo19q5KuUgEOO00c8EtLAQaNjRPJDkfXQ2TPXvs4qNTTwVeey3Y/FByYexdlYhS8NNP5gL7\n5JPAiBFB5ya5Bg2A2bOBE080dQyUX1iUROSTTZvMNFsutCecYKZs/Zx/GBiIfPLzz2ZaWRlsPmpr\n3br4T1NR7mJgIPKJVfFcVRVsPmrj5ZdNvifF7USfchUDA5HPjjkm6BykrmNHM73zzmDzQf7iU0lE\nPnD2hZRNH++VK4FDDjHpbMp3vvH6qSTeMRBRQs2aBZ0DCgIDA5GPsq1IpnHjoHNAQWBRElGGbd8O\n7LOPSVdWAvXqBZuf2qioMI3xAPNkUpi6Bycbi5KIssygQXY6m4ICYHe/DcT2DEu5i4GBKMM++yzo\nHNSd8w6Bdwv5g4GBKENuu83dMOzBB4PLixd27Qo6B+QXT+oYRGQwgPthAs0TqnpH1PozANwKoBpA\nFYDrVfU/CfbFOgbKetXVpthoyxagVSuzbP16oG3bYPNVF5s2AW3amHRFhbt4icIhdHUMIlIAYCKA\nkwB0AzBcRLpEbfauqvZQ1d8BuAjA5HSPSxRmv/xipvfeay/L1sFu9t/fTlv9PVFu86IoqQ+AUlX9\nVlX3AJgOYIhzA1V13oQ2BRA18i1RbrH6RSottZdlW8VzPOXlQeeA/OBFYGgLYJ1jfn1kmYuInCki\nSwG8AeAqD45LFFo7d5rpSy/Zy6zHPrPZ7t1B54D84Nt4DKo6A8AMETkGwNMAOifatri4+Nd0UVER\nioqKMp09Ik9ZdwxOe+3lfz68ZhWRUbBKSkpQUlKSsf2nXfksIn0BFKvq4Mj8GAAaXQEd9Z5vAPRR\n1e/jrGPlM2W9efOAfv3s+YKC7OpVNdq55wLTpgFz5gADBgSdG4oWuspnAAsAdBKRjiJSCGAYgFnO\nDUTkt450LwCIFxSIcoV1xzB2LLB4MbB6dbD5Sddhh5npxo3B5oP8kXZRkqpWichIALNhP666VEQu\nN6t1MoChInIhgAoAPwP4n3SPSxRm8+aZYTEdpaJZ7aqrgGeesceUoNzGvpKIMsBqJZxLH+UxY4Dm\nzYEbbww6JxQtjEVJROSQq6OdNW7Myud8wcBA5LF//zvoHGRG48bA+PHAd2yFlPMYGIg89OGHpn4B\nAP7612Dz4jVrbIYlS4LNB2UeAwORh44/3kwnTwbuuivYvHitS6Sjm9JSYOnSYPNCmcXKZyIPWZXO\nZWXAAQcEmxevOQccAnKrYj3bsfKZcsLGjcA//xl0Lry1ebOdbtIkuHxkyt57B50D8gsDAwXimWeA\nm24KOhfeWufoMSwXur+g/MXAQIGwOmO75hq70dSZZ2Z3he1FF9npXB2zoL5vvatRkFjHQL5atw74\n17/M0zsffmiKJ7ZvBz79FDjySKBHD+CLL4LOZd04h77M1Y9ws2Z2z7G5+jdmI9YxUCg98QRw8snA\n0KH2surq2I7jXnzR1C18+KGZt+4crGlhobnAZtv4wnPmAIceGnQuMq93bzvtrFOh3MLAQJ4YOxZ4\n6y3g5ZftZZdeChx0EPDKK3ajrx073O+zBn455hgzXbDAXjdpUnY0FlMFjj0WOOQQ4Ljjgs5NZj32\nmJ1euza4fFBmsSiJPHHAAXbPm9a/r1s34Ouv09932D8OlZWmTqFHD9MyuHdve4zkXKNquhC3bNoE\ntG4dXH7IYFEShdL27UHnIDhWcdnatUCrVrkbFIDYIj42dMtNDAyUNhFg1y73/Nq13twtAMCaNd7s\nJxMeeADo08ekf/zR3QAsHzj/75Q7GBgoLYmKeTp29O4YL7wA/P3v4StS2rbNPG67aJG9LB8agb37\nrp1mYMhNDAyUlsWLU9+2rs/AX3+9KbsvKDB3I6WldduP13r1il2WD4GhfXs7HW9sa8p+DAxUZ6rA\n5Zenvv3hh5tpo0a1CyjRFi6s+3u9FG+4zkaN/M+H35yVz7xjyE0MDFQrK1aYX+033mguEFYX07Wx\nZIkJEu3bA7/9rb38kENSe39YWt+GJR9+c1ZA//hjcPmgzGFgoLhE3L/MN20yj2N+842ZnzChdvsb\nMcJOH3ywma5dCyxfbkYF+/Zb00jshhuA+fOBJ59MvK+wdDcRHRiefTaYfPitsNBOx7trouzHwEAA\ngJUrgVWr3MucXVMsWmRep5xSt/2fdVb81sz16pnilw4dgP33N62i+/QBTjrJ9J00bFjse8LyS71e\nPff8uecGkw+/tW8PPP+8SVdUBJsXygwGBgJgunNwFusAQEkJ8OWXwHvvme4t6mLAAODmm4HTT6/d\nL/02bUyL6WnTgK1b3evCMrRkWAJUEKyK98rKYPNBmcGWzwTA/jVvnfq69FXUurW7/5ymTd1dYKxb\nZy7y8Z7mSaa8PH6lbmVl7K92P7VsCXz/vT2fTx/bykqgZ09TX1RdnX19W+Uatnym0BEBzjkn9o4g\nerCa9u1rHxQAU6bdsmXs8qCLMQry+NtTvz7w5z+bdD6fh1zlyb9URAaLyDIRWSEio+OsP1dEvoy8\nPhKRI7w4LnlPxB7bN5HoxmtvvGHKnMeNA265xV7u1ShmIsCQIbHL9+zxZv91UVZmF3E99VT2dhWe\njiDv1iiz0i4lFZECABMBDAKwAcACEZmpqsscm60CMFBVt4vIYACPAeib7rEpM5YvT75+wADzFBHg\nLkawBqpZtgx47TVT4eyVeGXZr74KnHeed8eojXbt7HRBgXliK984652qq3nnkEu8+Ff2AVCqqt+q\n6h4A0wG4ft+p6jxVtbpZmwegrQfHpYAceSTw2Wemi+x4ZcvPP29axN5zj3fHjBcYzj/fdN5X14px\nr+TrBdF53kfHlBNQNvPiI90WgGO0W6xH8gv/JQDe9OC45JMTTnDP77efqSs48kj/8mAFhqlT3cvP\nPjv4Io18DQzOQZjuvju4fJD3fH3gTkSOA3ARgGOSbVdcXPxruqioCEVFRRnNV74qKzNdGtTUZbbz\nwrdypRl8x28DBwJz58Y+Imp16LZ7t6n8zmSQKC83I88ddph7+RF5WmMWPTof+aekpAQlJSUZ23/a\nj6uKSF8Axao6ODI/BoCq6h1R23UH8BKAwar6TZL98XHVDFqxAmjYELjvPmDyZNPquCYnn2zGGZg6\nNfhHMl94wTwBFa2w0IwYN3Fi5o59ySVmCNNoQZ+ToCxeDHTvbs/n63kIA68fV/XijmEBgE4i0hHA\nRgDDAAx3biAiHWCCwgXJggJlXufOQNu25m4hVWEag9nqTiNaRUXd+m2qjXhBYeTIzB4zzI44whTl\nvfhi0Dkhr6VdOqqqVQBGApgNYAmA6aq6VEQuF5HLIpv9HUALAI+IyEIR+STd41LdWeMsJ2P9Ejz8\ncODUU7179DRdvXubX6bOrp8t0eNJ++Guu/w/Zpi88ELQOaBMYMvnPJPqL39VU8zUuLGZ/+kn05dS\nz56Zy1ttvPtubKV4mzbAhg2ZO2b0uduxw7TuznfWeSkrM2N/k//Y8pkyomlTU4a+117AH/5glllB\nAQCaNw9PUACAQYNiG5VZdwxffulPHz753FdSPOxpNXcwMBAA0+fNY48BO3cCL78cdG5qJhLbqGzn\nTjMEaM+eybvtrov582OXBf2YbNjk62O7uYj/yjw3ZIgpNurQIeiceGP8eDP1esjJvnHa6TMwuKXy\nhBtlBwaGPPfSS0HnIH3OgWMs69al/yTVsmXAmjWJ1/MXsvHBB8AxxwRT+U+ZwY92nlA13VhEy/Zf\nvarxL94rV6a/765dgaOPjr/u44/T33+uGDjQPCW2c2fQOSGvMDDkiYICdxcWAwYElxevtWkTu2zW\nLG/2vWVL/IZbnTp5s/9c0ayZuWP48MOgc0JeYGDIUZWVppsIAPjTn2LXjx4NzJzpb56y1fr1dvrB\nB820YcNg8hJWTZsCpaXm7oGyHwNDjrrySuA3vzHpKVNi1zduDJxxhr95yiSrG/BMuPJKO33BBWYa\nr14jnzVtCtx7r0kH3dstpY+BIUctWWI6yLOCQ7Rca0PYoQPwf/8Xu/zWW9Nv9Pbaa2Y6caI9Sh3v\nGNxWrLDTQY+sR+ljYMhBXbualsoA8MMP8bfJtcAAmD6got1yS+37hkqkQwc7IPCJJLdPP7XTVhEm\nZS9+vHPQsmXA2rXJt3G2as4VyYowdu1Kf/8iprVzLgbVdM2YYaf33Te4fJA3GBhylFXkEe2ee0wl\n4e9/729+/JDsgn3oocCmTTUXc1iNtBYujF3Hu4TEunUz3bNTbuBHPUd9/33ssnbtgOuuM49ahqUb\nbS/VVOnZpg1w2mlm6NF//CN2/bZtphfZa681I9RFY4VzchzFLXewd9UclOiiv2YN0LGjr1nxVUkJ\ncNxxwH//C/Trl3i7Jk1M0ZLzY1ZeDrRunXg0u+Ji0w8T7xqSKygw57W8nIHUT+xdleosl4MCYIrH\nnn665vqTePUN27YlDgovvACMHcugkIq33jJT1jNkN94x5KB4dwz5dEqjh5xMxPpVe955pu4h0Uhk\n+XTu0lVRYT+5xfPmH94xUFzz5wOPPx4bFBINhZnLUm1gZV3AnnuOw1N6xfnQw403Ap9wrMasxMCQ\nIy66CLj00tjlkyaZRl75pHt34OuvU9s22TP3Z5wR/+kkSsz5w2TCBOChh4LLC9Udx6DKctXVpny8\nqir++nbtgJtv9jdPQRMxjfxSkag+olEj9iXlBT9G0iPv8Y4hyz3+uOn2ItEXMJ/Lec88s+7v5aAz\n3mBgyE4MDFlu0yYzXbUq/vo9e/zLS9i88krt39OuXX4HUy88+6ydZmDITgwMWa6mRyjzvUOzggLg\nvvtMun1797p4Ty6demrm85TrGjWy0/n8wySbsY4hyyUKDFdfbb6UqZa15yqr7uXaa2NHqzvoIGDR\nIvcy9pqaPucdF+8YspMndwwiMlhElonIChEZHWd9ZxGZKyK7ReQ6L45JRqLA0LUr8PDDwF57+Zuf\nsGrXLrabi+ig+dBDwJgx/uUpVznrdt5+O7h8UN2lHRhEpADARAAnAegGYLiIdIna7HsAowDcle7x\nyC1RYLjsMn/zEXbr1plz0qyZvcz5RFK/fsDIkfGHCaXayfZxxMmbO4Y+AEpV9VtV3QNgOoAhzg1U\n9TtV/QwAbyw9ligw5GIneek66SR7nAoAuOIKM7qdKjB3bnD5ykVffWWnr7gC+O674PJCtedFYGgL\nYJ1jfn1kGfngb3+LXXYX78uSmjsXeP11oFUr0zCQvNetm52eNAn44IPg8kK1F8rK5+Li4l/TRUVF\nKCoqCiwvYeb8FXbXXaYTuf79Td8/lFiynlfJO+efDzzzjEl/9BEwdGiw+cklJSUlKCkpydj+0+5E\nT0T6AihW1cGR+TEAVFXviLPtWAA7VPXeJPtjJ3opOvFE4J13THrBAuDII4PND1E0Z5Hm1q1Ay5bB\n5SWXhbETvQUAOolIRxEpBDAMwKwk27P0Ow2vvw489pgZvtMKCt26MShQ+F17bdA5oFSlXZSkqlUi\nMhLAbJhA84SqLhWRy81qnSwirQF8CqAZgGoRuRrAYaq6M93j57rvvjOP/330kZkfNQpYvdq9TfPm\n/ueLqLZ++CHoHFCqPKljUNW3AHSOWjbJkd4MoH30+6hmS5cCH39sz9eP8x9ztjQlCqstW4LOAaWK\nXWKEnFVG+9FHwM6dQGmpe/24cWaEMaKw69Ej6BxQqkL5VBLZrHYKAwYAf/6zvXy//Uxl3uGHm95V\nicKoWzfTTuSSS4C1a4PODaWKdwwh52zA9uijdnrrVjNt2tTf/BDVxldfAUuWmOLOjz9OPG4IhQsD\nQ8jV1IKZfSFRNqhXzzxS/fLLQeeEUsHAEGKvv15zYyxWPFM2WLLETM85x9w5jBsXbH4oOQaGEHvz\nzcSDxlhjCzgHXycKqxtusNPHHAOMHRtcXqhmDAwhlqyXysWLzTTRmMVEYcIHJLILA0OIJRudbe+9\ngYULgU6d/MsPUV01aBA7mhufUgovBoYQq6niuWdPf/JB5IX69d1tGTiIT3gxMITQf/5jpt9/H3/9\nQQf5lxciLznb4nAwqfBiYAgZVWDQIGDHDuCpp2LXT50KvPWW//ki8kJhoXv+yy/Zh1IYMTAEbNo0\n4Oij7flffrGXxzN4MHDooZnPF1EmNGzonu/Z01RMd+9uel9lj/vhkPZ4DF7Lp/EYvvkG+PvfTRCw\n/uQtW4DWrRO/Z9s2U/FMlI3KyoB27RKvnzYNGDbMv/zkCq/HY2BfSQHq1Mnu0mLWLKBDB+Chh+Jv\n+8EHwMCB/uWNKBPatgX23x/YtCn+eqshHAUrbwLDKaeYlsQ1Penjt52RESmGDEm+HYMC5Yp584AD\nD4y/rqzM16xQAjlZx1BZ6Q4AqqYVcWWlP8fftg1YtSp2+d57A4884k8eiMKqY0d7LOhoX37pb14o\nvpwMDLt2malVbm/16OhXYBgxAvjtb+356moz4M5PPwHz5wPvvw9cc40/eSEKo+7d4y//8ktg+XKT\n/sc/gPPOq9v+r72WQSYdORkYrCd7qqvN1O/AEN3+4KWXgMMOM2kRYPJk4IEHUt/f0097lzeiMOjW\nDZg+PXZ5VRXQpYuZFhcDzz1XtyeV7r/fPNpNdZPVgaG6On5lbaLAEN0kP5P5cqZ37LDnP/sMePHF\n5O//3e/MADyAqaQ7/3zv80gUpIIC4MQTE693fmfuvhs44IDaH0MEuPhi4KabUn/PyJHet8h+7DHg\nj3/0dp+ZltWB4YcfgKuuil1uFSVZF2jrTqGmO4YtW9xFQLUxcyawYYNJO3/h1KtnKr0tX31Vcz5a\ntQL+9S+TTvboKlE2S9YB5I032unrrwc2bgTeead2+xcBnnwSmDgx9fc8/LD93auNzZsTP9gyZUr8\nxqphFvrA0LVr4g+EdQGOvmW07hisO4VUi5JWrHBXGldUmF/4c+Ykf19lJXDmmcDVV5t55x0DULfB\nSTjSFeW66MZuTs7RCi0nnhjb8Z5I4kdfrQv17t0msMTzyy/Azz+b73n0+6I99pi9rqzM/T1fvTr+\ne4DEY6Yk6vImDEIZGLZvt9PLlpkKqIqK2O2sD8Ts2aanUesXSPQdQ3RgqKoCPv/cva9Fi8wAIk7D\nhwNHHgkce6z5FR/P6tXAmjUm/fPPZvrjj0n/vLicv1L23bf27yfKNiLAgw/a86n8qh461Dz1d8EF\npogJMHf6lh077Dv3zZvNHXtFhSmKcl78LU2amLZERx7pzlc8c+ea6ccfm0Z6zvyWl9vTpUvN9ci6\nLjgDlHWcbduAli1Nurrafn9oqGqoXgD0+OP1V+a+QPXDD838nj2q1dWq69fb6047TXXKFJN2vuem\nm1QnTFDYy0LaAAAMTklEQVRdtcrMP/qoWT91qpkvLzfz1dX2ewDVnTtVL7tMtaDAvVxVddQo1Zdf\njs0foHrKKapbttjze/a41yd7qapu2KD6zTeq27erVlaqvveeEuW0V15xfwdS/b44X/37qy5bplpW\npnrWWYm3Kymxj3vLLaqffBL7HQRUhw416YULVc89V/XKK833cfjw2H3++KM59ttvm/n77rPXde6s\neuGF9vxVV5npm2+avEbva8SIms/XhAmqjzyi+sYbqqeeqrppk+rRR6uaS7mH12FPdgIMBrAMwAoA\noxNs8yCAUgBfAOiZZF8KqA4a5P6gHH+86oIFJj1mjPuEDhxop6uqYk9469Zm2q2b6k8/2UEEUL3z\nTtVXX03tA3j11Xb6m29U162L3eaQQ+x0v36p7bdDh5o/EES5aMYM8x1Ys8bMx/v+1ubVt2/idR98\nYI4xfryZd160AXPxdwaGW25xf9+HDk28b+sacs899rIDD1QtLIy//ciR8Ze3bq3avr3qO++Y69TK\nlaozZ5ofxj16xG5/5plWGqoeXMutV/o7MMVRKwF0BNAgcuHvErXNyQBej6SPBjAvyf7S+mCk8nrq\nqfT30b+/d/khyldWYHDK1Pe+Xz/V//1f1V69oi+q5mXdQZx1lh0kUn09/7yZTpqU2WtX4hdU1bvA\nkHYneiLSF8BYVT05Mj8mksk7HNv8C8D7qvp8ZH4pgCJV3Rxnfwqkl6ewO/54e8wFIPJvJcpDs2aZ\n7mCc34HoMv7o7wvF420nel5UPrcFsM4xvz6yLNk2ZXG2yXkzZwJHHGG+DJZBg4LLD1HQ4lX03nqr\neRLRanf073/7miVCaDvRK3akiyIv46KLzLPJ6WrY0LsnAUpKgKKimrc79VTgjDPcx33zTW/yQJSN\n4o0tcvPN7vl69fzJS9h17WqeeDJKIq/M8CIwlAHo4JhvF1kWvU37GrZxKE64plMn9/yxx5ouqWvL\ny8fDjjgi+fqlS80/1fqAN2hgr+OHnvJZ5875XZTavLnpQy0Vw4cDt9xizRXB+YMZ+IeX2fKkKGkB\ngE4i0lFECgEMAzAraptZAC4Efq2T2BavfiEVVnuCjh3NtG/f1N53yinu+WuvTe1969cnX9+5M9Ci\nBXDwwfHXL1hg+n5xNmYpKAC++AKYNCl83YAThdldd9XtfVOmxF/uxbjTf/pT3d9bU88GEybY6WQt\nxb2WdmBQ1SoAIwHMBrAEwHRVXSoil4vIZZFt3gCwWkRWApgE4Mpk+0z0z/rsM+Ccc0zaOkl77WUu\nzE4nnxz7XqsxDGAuxskCg3OENGtgkUS++MJMb7/dXjZ2rJ3u2dNMo/PYo4f5OxkYiJJzfkcOPNCM\negiY3lPXRWoup0wxDWNLS+Pvo22CGs2a7vZT8fjjdtpZGpAKqx5l5sz46y+4wE47W3iPH2/utPbb\nr3bHS5UnLZ9V9S1V7ayqh6jq7ZFlk1R1smObkaraSVV7qOrnifcWf+i/UaOAXr3MrdfKlcB775nl\n9erFNi23/lFHHWUv69rVTMeMMS0NrRMar1n+rbcC111nzzv/2Q89BIwebTrnAuzm7qedZqYTJrgD\nCYuKiOpu9GjzXa2qMt1XDB0KjBtn1jVrZn/XGjc21warqPmGG0ygGD8e+PbbxN1SHHig3SLa2T+T\n9cPT6hW5JtZDo+0dBebWNWbcOBOwnD9OLVZgaNLETJs2tVtuAyagWdczZ48QVseA0d3veMbLZ1+9\neAHQxYtVt25Vfe45+zndeADVG290P/t8wAH2/NSpqg0amBbMqqq//GKeT7bcequ7Qc2nn5rpgw+q\n/uUv9nEPPtjeZvZs+/1bt7rz8/33Zv9bt9qNaIjIe4BpFGf1WhDdG8FNN7m3txrHDhjgfv5/1iyz\nftcu87KWv/++mTobwZ12mr3//v1NA9donTqZ9dXVZn7OHNOoVlV1xw5zvCOOsPfZqpXqRx/Z16Gm\nTe1jWNeP6mrTe8OIEWbZpZfax2vRIjPtGELZV9Lhh5t+RIYPr3lbq18ka4jM3bvtdV27AosXA19/\nbeYbNXL/gr/5ZlPeb+nd20yjK8PuvNO8AHc5n9XXiaVFC7P/li1NRM/nSjWiTCotNfWMVjFTYaF7\nfXQnlL17mw4ynd/3gQPt73zjxu7SgwEDzPSvf7W79nd2wlmvXvySDWv/Vr4GDDB3NoC5Gzj9dNMv\n265dpnvvGTOA/v3t91nXDGfHoCLAQQeZ7vf/+EcznoulTRv7bsNLIX1c1XbSSfH/AYBp9GKVEe61\nl3lk1LpY1+aiHD04uSpw1lmmAz/A3L4CpvtfPyuAiCi+6KcTo8v2o3tSti6u1gX4D3+I7fW4oAC4\n8ELTbkLEdMjXtKm5xowa5S7PT3R9mTIltXGrGzeOHY9i9mw7OBXE+ck+aFBsu6c5c0xxVLJ60LoI\nfWB4663E6447zj3//vt1O0b0h6p1a+D3vwdeey1220SVWEQUnOjvcI8e8bezSgwSdYXv/KXetKmd\n3rDBfiilTx/TJime/v1rzmsiJ5xgp1N9KCX6oRavhD4w+GHOHLsIautW4De/ib8di4aIwqm+40qW\n7Ht63nmJK6KTadPGTs+fX/v311a8OwY/pd1XktdERMOWJyIKLxEzVkK/fkHnxDvl5aZbEOtpx5qI\nhK+vJCKiQNW2/UDYNWyYelDIBAYGIsp61pM/5A3WMRBRVvvmm8Rd0lDdsI6BiCjLsY6BiIgyioGB\niIhcGBiIiMiFgYGIiFwYGIiIyIWBgYiIXBgYiIjIhYGBiIhcGBiIiMiFgYGIiFwYGIiIyIWBgYiI\nXNIKDCKyr4jMFpHlIvK2iOydYLsnRGSziCxK53hERJR56d4xjAHwrqp2BvAfADck2O5JACeleSyq\npZKSkqCzkFN4Pr3F8xle6QaGIQCs4bOnAjgz3kaq+hGAH9M8FtUSv3je4vn0Fs9neKUbGFqp6mYA\nUNVNAFqlnyUiIgpSjSO4icg7AFo7FwFQADfH2Zwj7BARZbm0RnATkaUAilR1s4jsD+B9Ve2aYNuO\nAF5V1e417JPBhYiolrwcwS3dMZ9nARgB4A4AfwQwM8m2Enkl5eUfR0REtZduHcMdAE4QkeUABgG4\nHQBEpI2IvGZtJCLPAZgL4FARWSsiF6V5XCIiypC0ipKIiCj3hKbls4gMFpFlIrJCREYHnZ9sISJr\nRORLEVkoIp9EliVseCgiN4hIqYgsFZETg8t58OI1vKzLuRORXiKyKPLZvd/vvyMsEpzPsSKyXkQ+\nj7wGO9bxfCYgIu1E5D8iskREFovIVZHl/nw+VTXwF0yAWgmgI4AGAL4A0CXofGXDC8AqAPtGLbsD\nwPWR9GgAt0fShwFYCFO3dGDknEvQf0OA5+4YAD0BLErn3AGYD+CoSPoNACcF/beF6HyOBXBdnG27\n8nwmPZf7A+gZSTcFsBxAF78+n2G5Y+gDoFRVv1XVPQCmwzSeo5oJYu/8EjU8PAPAdFWtVNU1AEph\nzn1e0vgNL2t17iJP4zVT1QWR7Z5CgoaeuS7B+QTiP3QyBDyfCanqJlX9IpLeCWApgHbw6fMZlsDQ\nFsA6x/z6yDKqmQJ4R0QWiMglkWWtNX7Dw+jzXAae52iJGm0mOndtYT6vFn52Y40UkS9E5HFH0QfP\nZ4pE5ECYO7F5qP13u07nMyyBgequv6r2AnAKgP8VkQGIbWjIJwzqjucuPY8AOFhVewLYBOCegPOT\nVUSkKYAXAVwduXPw5bsdlsBQBqCDY75dZBnVQFU3RqZbAcyAKRraLCKtASByK7klsnkZgPaOt/M8\nx6rtueM5TUJVt2qkcBvAY7CLLnk+ayAi9WGCwtOqarUR8+XzGZbAsABAJxHpKCKFAIbBNJ6jJESk\nSeQXBURkLwAnAlgMu+Eh4G54OAvAMBEpFJGDAHQC8ImvmQ6f6IaXtTp3kdv57SLSR0QEwIVI3tAz\n17nOZ+TiZTkLwFeRNM9nzaYA+FpVH3As8+fzGXTtu6MWfjBMzXspgDFB5ycbXgAOgnmCayFMQBgT\nWd4CwLuR8zkbwD6O99wA88TCUgAnBv03BHz+ngOwAUA5gLUALgKwb23PHYDekfNfCuCBoP+ukJ3P\npwAsinxOZ8CUkfN81nwu+wOocny/P49cI2v93a7L+WQDNyIicglLURIREYUEAwMREbkwMBARkQsD\nAxERuTAwEBGRCwMDERG5MDAQEZELAwMREbn8P0l/MnVdI5SHAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "b = np.convolve(c, x0) \n", + "b += np.linalg.norm(b)/20/np.sqrt(2*n-1)*np.random.randn(2*n-1)\n", + "plt.plot(b)" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "----------------------------------------------------------------------------\n", + "\tSCS v1.2.6 - Splitting Conic Solver\n", + "\t(c) Brendan O'Donoghue, Stanford University, 2012-2016\n", + "----------------------------------------------------------------------------\n", + "Lin-sys: sparse-indirect, nnz in A = 816002, CG tol ~ 1/iter^(2.00)\n", + "eps = 1.00e-03, alpha = 1.50, max_iters = 2500, normalize = 1, scale = 1.00\n", + "Variables n = 1001, constraints m = 3001\n", + "Cones:\tlinear vars: 1000\n", + "\tsoc vars: 2001, soc blks: 1\n", + "Setup time: 4.53e-02s\n", + "----------------------------------------------------------------------------\n", + " Iter | pri res | dua res | rel gap | pri obj | dua obj | kap/tau | time (s)\n", + "----------------------------------------------------------------------------\n", + " 0| 1.22e+00 8.17e-01 9.22e-01 -1.16e+01 2.51e-01 3.49e-15 7.49e-02 \n", + " 100| 1.94e-03 5.57e-06 1.58e-05 2.29e-01 2.29e-01 1.39e-15 2.84e+00 \n", + " 200| 8.96e-04 3.02e-06 3.95e-05 2.29e-01 2.29e-01 1.39e-15 5.57e+00 \n", + "----------------------------------------------------------------------------\n", + "Status: Solved\n", + "Timing: Solve time: 5.57e+00s\n", + "\tLin-sys: avg # CG iterations: 10.08, avg solve time: 2.75e-02s\n", + "\tCones: avg projection time: 4.08e-06s\n", + "----------------------------------------------------------------------------\n", + "Error metrics:\n", + "dist(s, K) = 4.0485e-16, dist(y, K*) = 0.0000e+00, s'y/|s||y| = 8.2313e-17\n", + "|Ax + s - b|_2 / (1 + |b|_2) = 8.9635e-04\n", + "|A'y + c|_2 / (1 + |c|_2) = 3.0178e-06\n", + "|c'x + b'y| / (1 + |c'x| + |b'y|) = 3.9507e-05\n", + "----------------------------------------------------------------------------\n", + "c'x = 0.2293, -b'y = 0.2294\n", + "============================================================================\n" + ] + }, + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEACAYAAABRQBpkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF91JREFUeJzt3X+UXGWd5/H3Nwk/BCRmRtNZxAEBicg5/IhOBmU4lJOA\niCMJZ5VlZIcEht0Zxxk8iiyJO2fpnj9EdDwsc1x2xR/YMGE0MkKix5UYQ+ERVuVH+GECITP8/pEG\nDUZxOBCa7/5xL0mn0y1p6t5UV9X7dU5O7n266j5PPVXVn/s8T92uyEwkSb1tSrsbIElqP8NAkmQY\nSJIMA0kShoEkCcNAkkRFYRARSyNiXUTcExHLImLPiJgREasiYkNE3BgR06uoS5JUvZbDICIOAv4L\ncGxmHgVMA/4MWAKszszZwBpgaat1SZLqUcXI4NfAi8C+ETENeB3wBLAAGCxvMwgsrKAuSVINWg6D\nzHwW+ALwKEUIbMnM1UBfZg6Vt9kEzGy1LklSPaqYJjoE+ARwEHAAxQjhLGD037nw715I0iQ1rYJj\nvAu4JTM3A0TE9cB7gKGI6MvMoYiYBTw91p0jwpCQpNcgM6OqY1WxZrABOC4i9o6IAOYB64GVwOLy\nNouAFeMdIDP9l8nFF1/c9jZMln/2hX1hX/zuf1VreWSQmXdHxNXAHcAwsBa4Eng9sDwizgUeAc5o\ntS5JUj2qmCYiMz8PfH5U8WZgfhXHlyTVyyuQJ5FGo9HuJkwa9sV29sV29kV9oo65pwk1ICLb3QZJ\n6jQRQU6yBWRJUoczDCRJhoEkyTCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiS\nMAwkSVQQBhFxeESsjYg7y/+3RMT5ETEjIlZFxIaIuDEiplfRYEmda86X5rDmoTXtbobG0HIYZOYD\nmXlsZs4B3gn8FrgeWAKszszZwBpgaat1Sepsazet5Qf/9oN2N0NjqHqaaD7wb5n5GLAAGCzLB4GF\nFdclSapI1WHwn4Bry+2+zBwCyMxNwMyK65IkVWRaVQeKiD2A04CLyqLR32U57ndb9vf3b9tuNBp+\nz6kkjdJsNmk2m7Udv7LvQI6I04C/zsxTyv37gEZmDkXELOCmzDxijPv5HchSj4iBYMnxS7hk/iXt\nbkrHm8zfgfxnwD+P2F8JLC63FwErKqxLklShSsIgIvahWDz+9ojiS4GTImIDMA/4bBV1SZKqV8ma\nQWb+O/CmUWWbKQJCkjTJeQWyJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaS\nJAwDSRKGgSQJw0CShGEgScIwkCRR3TedTY+Ib0XEfRGxLiL+KCJmRMSqiNgQETdGxPQq6pIkVa+q\nkcHlwPfKL7w/GrgfWAKszszZwBpgaUV1SZIq1nIYRMT+wAmZeRVAZr6UmVuABcBgebNBYGGrdUmS\n6lHFyOCtwC8i4qqIuDMiroyIfYC+zBwCyMxNwMwK6pLU42IgyMx2N6PrTKvoGHOAj2Xm7RFxGcUU\n0ehna9xnr7+/f9t2o9Gg0WhU0CxJ6h7NZpNms1nb8aPVhI2IPuD/ZeYh5f4fU4TBoUAjM4ciYhZw\nU7mmMPr+acpLvSEGgiXHL+GS+Ze8pvtnJlP+fgrD/2OYKdHbH4aMCDIzqjpey71ZTgU9FhGHl0Xz\ngHXASmBxWbYIWNFqXZKkelQxTQRwPrAsIvYAHgTOAaYCyyPiXOAR4IyK6pLUo7Kcbc5MqOycWFBR\nGGTm3cAfjvGj+VUcX5KAbQvHOf4SpF6j3p50kyQBhoGkDrLDNJEqZRhIkgwDSZ3DNYP6GAaSJMNA\nUudwzaA+hoGkjuE0UX0Mgx5z62O3csP9N7S7GZImGcOgx5z17bM4/Zunt7sZ2kVfW/s1bn/y9nY3\nY9Jwmqg+hoE0if3Fyr/gotUXtbsZ6gGGgaSO4ZpBfQwDSZJhIKlzuGZQH8NAUsdwmqg+hoEkyTCQ\n1DmcJqpPJV9uExEPA1uAl4GtmTk3ImYA3wQOAh4GzsjMLVXUJ0mqVlUjg5eBRmYem5lzy7IlwOrM\nnA2sAZZWVJekHuWaQX2qCoMY41gLgMFyexBYWFFdaoHDa0ljqSoMEvhBRNwWEeeVZX2ZOQSQmZuA\nmRXVJalHuWZQn0rWDIDjM/OpiHgTsCoiNsBO47hxn73+/v5t241Gg0ajUVGzNFpEtLsJ0mvWy9NE\nzWaTZrNZ2/ErCYPMfKr8/5mIuAGYCwxFRF9mDkXELODp8e4/MgwkSTsbfaI8MDBQ6fFbniaKiH0i\nYr9ye1/gZOBeYCWwuLzZImBFq3VJ6m1OE9WnipFBH3B9RGR5vGWZuSoibgeWR8S5wCPAGRXUJUmq\nQcthkJkPAceMUb4ZmN/q8SXpFb28ZlA3r0CWJBkGvca5VnUy1wzqYxhI6hhOE9XHMOgxXmcgaSyG\nQY9xeK1O5jRRfQwDSZJh0GucJlInc82gPoaBJMkwkNQ5XDOoj2HQY3wTqZM5TVQfw0CSZBj0GheQ\n1cmcJqqPYdBjfBNJGktXh8HVd1/NU795qt3NkFQR1wzq09VhsOiGRXzxZ19sdzMmFaeJJI2lsjCI\niCkRcWdErCz3Z0TEqojYEBE3RsT0quqS1JtcM6hPlSODjwPrR+wvAVZn5mxgDbC0wrok9SCniepT\nSRhExIHAqcBXRhQvAAbL7UFgYRV1qTWeUUkaS1Ujg8uAC2GHuO7LzCGAzNwEzKyoLkk9ymmi+rQc\nBhHxAWAoM+8CftfqpM/eJOACsqSxTKvgGMcDp0XEqcDrgNdHxDXApojoy8yhiJgFPD3eAfr7+7dt\nNxoNGo1GBc3SWDyjUifr5TWDZrNJs9ms7fgth0Fmfhr4NEBEnAhckJl/HhGfAxYDlwKLgBXjHWNk\nGEiSdjb6RHlgYKDS49d5ncFngZMiYgMwr9xXmzlNpE7mmkF9qpgm2iYzbwZuLrc3A/OrPL5a55tI\nnayXp4nq1tVXIEuSdo1hIKljOE1UH8NAkmQY9BoXkNXJXDOoj2HQYxxeSxqLYSCpY7hmUB/DoMc4\nTaRO5jRRfQyDHuMZlaSxGAaSOobTRPUxDCRJhoGkzuGaQX0MA0mSYSCpc7hmUB/DQFLHcJqoPoZB\nj/E6A0ljMQx6jMNrdTKnierTchhExF4R8dOIWBsR6yLiM2X5jIhYFREbIuLGiJjeenMlSXVoOQwy\n8wXgvZl5LHAU8CcRcTywBFidmbOBNcDSVuuS1NtcM6hPJdNEmfnv5eZe5TGfBRYAg2X5ILCwirok\nSdWrJAwiYkpErAU2Ac3MXA/0ZeYQQGZuAmZWUZek3uWaQX2mVXGQzHwZODYi9gdujIgG7DSO89mT\n1BKniepTSRi8IjN/HRHfA94FDEVEX2YORcQs4Onx7tff379tu9Fo0Gg0qmyWJHW8ZrNJs9ms7fgt\nh0FEvBHYmplbIuJ1wEnAALASWAxcCiwCVox3jJFhIEnj6eVpotEnygMDA5Uev4qRwX8ABqO4mmkK\ncE1m/rBcQ1geEecCjwBnVFCXJKkGLYdBZt4LzBmjfDMwv9XjS9IrXDOoj1cgS5IMA0mdo5fXDOpm\nGEjqGE4T1ccwkCQZBpI6h9NE9TEMJEmGgaTO4ZpBfQwDSZJhIKlzuGZQH8NAUsdwmqg+hoEkyTCQ\n1DmcJqqPYSBJMgwkdQ7XDOpjGEiSDANJncM1g/q0HAYRcWBErImIdRFxb0ScX5bPiIhVEbEhIm6M\niOmtN1dSL3OaqD5VjAxeAj6ZmUcC7wY+FhFvB5YAqzNzNrAGWFpBXZKkGrQcBpm5KTPvKrefA+4D\nDgQWAIPlzQaBha3WJam3OU1Un0rXDCLiYOAY4CdAX2YOQREYwMwq65pAm9pRrSR1lGlVHSgi9gOu\nAz6emc9FxOjoHjfK+/v7t203Gg0ajUZVzfIMQuoivbxm0Gw2aTabtR2/kjCIiGkUQXBNZq4oi4ci\noi8zhyJiFvD0ePcfGQaSpJ2NPlEeGBio9PhVTRN9DVifmZePKFsJLC63FwErRt9JkibCNYP6tDwy\niIjjgbOAeyNiLcV00KeBS4HlEXEu8AhwRqt1SeptvTxNVLeWwyAzbwGmjvPj+a0eX5JUP69AltQx\nnCaqj2EgSTIMJHUO1wzqYxhIkgwDSZ3DNYP6dH0Y+OcopO7hNFF9uj4MPIOQJhdP0Canrg8DSZPD\ntrP6Fk7QnCaqj2EgTXKBZ9Kqn2EgTXLdMj++7ay+hcfjmkF9uj4MnJ+UpFfX9WHg3KI0ObhmMLl1\nfRhI6h5OE9XHMJC0W1SxZqD6GAaSOobTRPUxDCTtFlWsGag+lYRBRHw1IoYi4p4RZTMiYlVEbIiI\nGyNiehV1vYa2taNaSTVwzaA+VY0MrgLeN6psCbA6M2cDa4ClFdU1IZ6FSJODawaTWyVhkJk/Bp4d\nVbwAGCy3B4GFVdQlqXe5ZlCfOtcMZmbmEEBmbgJm1liX1LW65c9RVHKdgdNEtZm2G+sa99nr7+/f\ntt1oNGg0GruhOVJn8BefAJrNJs1ms7bj1xkGQxHRl5lDETELeHq8G44MA0ndqZK/TdTD00SjT5QH\nBgYqPX6V00RR/nvFSmBxub0IWFFhXa/K4aQk7bqqPlp6LXArcHhEPBoR5wCfBU6KiA3AvHJ/t+nl\nMwhpMnLNYHKrZJooMz8yzo/mV3H818IXjSTtOq9AlrRbuGYwuXVtGPiikbqPI/76dG8Y+KJRl+iW\n6wxe4Qna5NS9YeDIQF2iW05oqngv+r6uT/eGgSMDaVLyPTk5dW8YeAahLtEt00RVhIAnefXp2jDQ\n2HwTdR6fM+0OXRsGnkFIk0slF5054q9N94aBL5oxdcuUg3qTJ3n16d4w8EUjTSp+uc3k1r1h4MhA\n6jq+r+vTtWGgnT3+ODzzTLH91FPtbYt6TxVrBqpP14aB00Q7+u1v4eSTYVr5pwnf/W547rn2tkma\nKN/X9eneMHA4uYNrr4VDD4UZM4r9E06AL3yhvW3SrumWRX/XDCa3rgyDl1+GZdcWL7jVP0w+9znY\nurXNjWqjTLjiCvjbv93+RlyyBK68El56qc2N004y4etfhzPPLPbvuCP5yleK8l7nSV59ui4MMuEv\n/xK+/OXixTJrVrJyJZx9NgwPt7lxbfKTnxRTQvNHfLvEkUfCW94C3/9++9qlnQ0Pw1/9FVx+OZx6\nalF28MHwj/9YTPNt3tzW5rXkhReK9+RzzyU33QTLlsHtt8O998KPfwx33gnPPz/2fX/zG7j1Vrjv\n/uIY69YnW7YYkFWqPQwi4pSIuD8iHoiIi+qub9ky+NnPYOXKYv8d74DVq+EXv4DzzuvNF88VV8BH\nPwpTpuw45XDOOUV/aXIYHoYzzoAHH4Qf/ag4gQH4/d8P7rgDjjkG3vMeePTR9rZzVwwPw4YN8KUv\nFa+zefPg7UcUP7vuX2DpUvjud2HRomIEdNFFsHhxMY15wAFw1FHF9hFHwOzZ8KY3wbnnwtVXF8e4\n5pqi7LDD4JOfhLvuKmYE9NpV8k1n44mIKcAXKb728kngtohYkZn311Hfs8/ChRfCihWw777b5yf3\n3rsoO/bY4kz4/e+vo/bJ6cUX4Tvf2b4+MHK+9oMfLN6UW7fCHnu0qYHa5pJL4Omn4Yc/hD333F6e\nJHvsAZ//fPGL8oQTYNWq4pfkZLN1a/F6u+SS4hNrJ54IjQZ85CPQd3By9LVw9tnJ//nTse8/PFzc\n74knipHrL39ZTGUeeWTRJ9etTz78LfjMZ5IP3QD33APXX1+MoiKKqc8PfGC3PuSuUWsYAHOBjZn5\nCEBEfANYANQSBkuXwumnw9y5sPn5HecW99kH/uEf4IIL4KSTtn+qptvdfHNxdjVz5s4/O+AAOOQQ\nuOWW4g2r9nnoIbjssuKX28ggGO0Tn4Dp0+G97y2e27e9bfe1cTybN8PPf158SGH58mI0/qlPwYc/\nXIxGt91unCmgkaZOhQMPLP5B8RodTwQcfXTx7+KLodksRhfnnQd/93fFz7Xr6v6V+GbgsRH7j1ME\nxA5O7O8f9wC7+nxu3Qp3PwF/8zHob8LzW4tX3m1P3kZ/szh+7gfPHwdzLoDfm7GLB+5wDz4Es/+0\n6BOAx3/9OMC2Pnn9B+Hsq+CQZluap9LGf4WjzocvbwQ27vizB375wLbnC4BD4J0XwLs+VUwdJUAW\n/2eW2zmifALb2+6/C9vPPVdMv740DG98Ixz+Nlj8ddh/f7gP+Psf7fg4xnpPTtT6Z9YDsHz9ctY9\ns27HHwZ86Ivwv78B//RRmDVr/OOYEzuLOlflI+I/Au/LzP9a7v9nYG5mnj/iNnnQiSduu88bDj6Y\nN7z14GJngk079NBiaPmKF4dfZM+pO55m/epXcNfdEz92p5o6tRgp7bXX9rKR/fLii8Uai58qaq+9\n9iqep6lTdyx/4aUX2Hva3mN+HHPjRnjiyeIX2ytnwRG7b3uffYoR57777vpZ+FjvyYnaOryVPaaO\nP6+5dWuxMP3Ci+PcoEPf+796+GF+9fDDQPEQHr35ZjKzslyrOwyOA/oz85RyfwmQmXnpiNukHxOT\npImJiErDoO5PE90GHBYRB0XEnsCZwMqa65QkTVCtawaZORwRfwOsogier2bmfXXWKUmauFqniXap\nAU4TSdKEddo0kSSpAxgGkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAM\nJEkYBpIkDANJEi2GQUR8KCJ+HhHDETFn1M+WRsTGiLgvIk5urZmSpDq1OjK4FzgduHlkYUQcAZwB\nHAG8H7giYle/Mrt3NZvNdjdh0rAvtrMvtrMv6tNSGGTmhszcCIz+Rb8A+EZmvpSZDwMbgbmt1NUL\nfKFvZ19sZ19sZ1/Up641gzcDj43Yf6IskyRNQtNe7QYR8QOgb2QRkMB/z8zv1NUwSdLuE1V8GX1E\n3ARckJl3lvtLgMzMS8v97wMXZ+ZPx7hv6w2QpB6UmZWtxb7qyGACRjZqJbAsIi6jmB46DPjZWHeq\n8sFIkl6bVj9aujAiHgOOA74bEf8XIDPXA8uB9cD3gL/OKoYgkqRaVDJNJEnqbG29AjkiTomI+yPi\ngYi4qJ1tqVtEHBgRayJiXUTcGxHnl+UzImJVRGyIiBsjYvqI+3T1hXsRMSUi7oyIleV+T/ZFREyP\niG+Vj21dRPxRD/fF0rIP7omIZRGxZy/1RUR8NSKGIuKeEWUTfvwRMafswwci4n/uUuWZ2ZZ/FEH0\nr8BBwB7AXcDb29We3fB4ZwHHlNv7ARuAtwOXAv+tLL8I+Gy5/Q5gLcW6zsFlX0W7H0fFffIJ4J+A\nleV+T/YF8HXgnHJ7GjC9F/ui/F3wILBnuf9NYFEv9QXwx8AxwD0jyib8+IGfAn9Ybn8PeN+r1d3O\nkcFcYGNmPpKZW4FvUFys1pUyc1Nm3lVuPwfcBxxI8ZgHy5sNAgvL7dPo4gv3IuJA4FTgKyOKe64v\nImJ/4ITMvAqgfIxb6MG+AH4NvAjsGxHTgNdRXKPUM32RmT8Gnh1VPKHHHxGzgNdn5m3l7a4ecZ9x\ntTMMRl+Y9jg9cmFaRBxMkf4/AfoycwiKwABmljfr9gv3LgMupLhm5RW92BdvBX4REVeVU2ZXRsQ+\n9GBfZOazwBeARyke15bMXE0P9sUoMyf4+N9M8fv0Fbv0u9W/WrqbRcR+wHXAx8sRwugV/K5f0Y+I\nDwBD5Ujpd320uOv7gmKIPwf4X5k5B/gtsITefF0cQjF1eBBwAMUI4Sx6sC9eRS2Pv51h8ATwByP2\nDyzLulY59L0OuCYzV5TFQxHRV/58FvB0Wf4E8JYRd++m/jkeOC0iHgT+GfiTiLgG2NSDffE48Fhm\n3l7u/wtFOPTi6+JdwC2ZuTkzh4HrgffQm30x0kQf/2vql3aGwW3AYRFxUETsCZxJcbFaN/sasD4z\nLx9RthJYXG4vAlaMKD+z/DTFW/kdF+51msz8dGb+QWYeQvG8r8nMPwe+Q+/1xRDwWEQcXhbNA9bR\ng68Lig9VHBcRe5d/5XgexbVKvdYXwc4X8S4ut1/18ZdTSVsiYm7Zj2ePuM/42rxyfgrFC2AjsKTd\nK/k1P9bjgWGKT02tBe4sH//vAavLflgFvGHEfZZSfELgPuDkdj+GmvrlRLZ/mqgn+wI4muLk6C7g\n2xSfJurVvriQIgzvoVgs3aOX+gK4FngSeIFi7eQcYMZEHz/wToqvGNgIXL4rdXvRmSTJBWRJkmEg\nScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCfj/2fA+McUfZU4AAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "x = cvx.Variable(n)\n", + "f = cvx.sum_squares(cvx.conv(c, x) - b)\n", + "prob = cvx.Problem(cvx.Minimize(f), [x >= 0])\n", + "prob.solve(solver=cvx.SCS, verbose=True)\n", + "plt.plot(x.value)\n", + "plt.plot(x0)" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEACAYAAAC3adEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmAzfX+x/Hne5jFjD2ibNEUMYQytLkTWSLpaqNbSYqK\n1qvQxm37pV0p3bbbjvaUZSRGJdkpsiaMJRFZhmGWz+8PQzMa61m+Z3k97j33nnPme76f13ydmdd8\n12POOURERPaJ8TqAiIiEFhWDiIgUoWIQEZEiVAwiIlKEikFERIpQMYiISBF+KQYza29mi81sqZn1\nP8g0aWY218wWmNlkf4wrIiL+Z76ex2BmMcBSoDWwDpgJdHXOLS40TTnge6Ctc26tmVVyzm3yaWAR\nEQkIf6wxpALLnHOrnHM5wEig8wHTXAV87JxbC6BSEBEJXf4ohmpAZqHHawqeK+xUoKKZTTazmWZ2\njR/GFRGRACgZxHGaAq2AJGCamU1zzi0P0vgiInKE/FEMa4GahR5XL3iusDXAJudcNpBtZt8ApwN/\nKwYz08WbRESOknPO/DUvf2xKmgkkm1ktM4sDugKjD5jmc+BcMythZolAc2DRwWbonNPND7dBgwZ5\nniGSblqeWp6hevM3n9cYnHN5ZtYXmMDeonndObfIzHrv/bJ7xTm32MzSgR+BPOAV59zPvo4tIiL+\n55d9DM658UDdA5777wGPnwKe8sd4IiISODrzOYKlpaV5HSGiaHn6l5Zn6PL5BDd/MzMXaplEREKZ\nmeFCbOeziIhEEBWDiIgUoWIQEZEiVAwiIlKEikEkwmTtySI3P9frGBLGVAwiEWLKyimcPrwJxw2p\nzHFDKjFo8iAVhBwTHa4qEgE+WPghN3x8K/lfvMTJOZewemsm8Zf35KzGFfm420hiTH8DRjIdrioi\nRcxaN4seH/ahylfpzHmvC/PnxbB2YS06bBnL19PX8/Dkx72OKGFGawwiYSxrTxbJTzXFJj/MglFX\nULHiX19zDjpfs4avTm7Mgtunc3LFk70LKgGlNQYR2e+BcU+x+efGjH+6aCkAmMG7L1UndmY/eo66\n25uAEpZUDCJhav329bw463luPmUIjRoVP03ZsvD0FbfxfeZUFv6uCxrLkdGmJJEw9c9X+zBpQil+\ne/spSpU6+HT5+VDlskc5o+1yxt/0v+AFlKDRpiQRYWPWRr5c9T4Pte93yFIAiImB+9r35uu1n7I1\ne2twAkpYUzGIhKGBn75EworLuOXaqkc0/c3dKxGzsg3DpowIcDKJBCoGkTCTnZvNO4tf4s4W/yY2\n9sheEx8P7Y/vyUs/vBHYcBIRVAwiYeblKZ/i1p/OwBvrHdXrHvjXBfy2ayUrNq8MTDCJGCoGkTDz\n3JTXaVOp52H3LRzozKYlKbu+M0MnfBKYYBIxVAwiYWTJ77+yes98Hu9+yTG9/sJaXfj4ZxWDHJqK\nQSSMDPzgDar9cRUN68cf0+vvvKQ163N/Zt229X5OJpFExSASJvJdPuPWvcUtZ/U85nmc2SSOUuva\n8FpGuh+TSaRRMYiEibE/TmPP9rLcduVBTnM+AmZwVpW2fDxvgh+TSaTxSzGYWXszW2xmS82sfzFf\n/4eZ/Wlmcwpu9/tjXJFo8uS4UaTYlSQl+Tafq89qw6LdX5Hv8v0TTCKOz8VgZjHAMKAd0ADoZmbF\nHUf3jXOuacHtEV/HFYkmefl5TNv6IX3TrvB5Xpe1qUnu9kpM/WWeH5JJJPLHGkMqsMw5t8o5lwOM\nBDoXM53fruMhEm1GTvuW/B1VuK5TXZ/nlZQE1Xa15bUMbU6S4vmjGKoBmYUeryl47kBnmdk8Mxtj\nZvX9MK5I1Hj+6w9olnDlEZ/pfDitTrqASb9O9M/MJOKUDNI4s4GazrmdZnYh8Blw6sEmHjx48P77\naWlppKWlBTqfSMjKzc9lzq6PeeeCaX6bZ4/W5/Lul1eRk5dDbAk/tY0ETUZGBhkZGQGbv8+X3Taz\nFsBg51z7gscDAOecG3KI1/wKnOGc21zM13TZbZFCRv3wDVe/ezvZQ+dSooR/5pmXB3G3N2LsTa/T\nLqWZf2YqngnFy27PBJLNrJaZxQFdgdGFJzCzKoXup7K3kP5WCiLyd8MnjaZhfGe/lQJAiRJQPf9c\nRn7/rf9mKhHD52JwzuUBfYEJwEJgpHNukZn1NrNeBZNdZmYLzGwu8Bxwpa/jikQD5xzTt37OdS0u\n9vu8W5x4Ht+t/s7v85Xwp09wEwlh3y5exD9ea8vOR1aTkODfA/s++iqTbpPPYM+jGzDTQYPhLBQ3\nJYlIgDw7djQn517s91IAuOi8GuRlJzI3c6nf5y3hTcUgEsImrxvN5Y38vxkJICEBKu08l/e+1eYk\nKUrFIBKilq3bwJ+xC7nrn2kBG+P0Ss3JWDYjYPOX8KRiEAlRz3w5hqpZbalU4dgusX0k2tRvxvKd\nMwM2fwlPKgaREDX2l9G0rRWYzUj7XH5eY7bFLWbnnl0BHUfCi4pBJATtyskms+Qk+ra/MKDj1K6R\nQOyfpzF2ri6oJ39RMYiEoLczviNuawOaNTgu4GNVt1S+mKP9DPIXFYNICHp/ejqnJ7YPylhNqzZj\nxhrtZ5C/qBhEQtDsreO54ox2QRmrfcNUVuZojUH+omIQCTFL1q0lq8Q6buwQnIvbdTnvNLJj17Nx\n+5agjCehT8UgEmJeGJdOlawLKFvGj1fNO4SKFUpQ6s8mfDp9dlDGk9CnYhAJMeOWpZNWPTj7F/ap\nGdeEiT/ND+qYErpUDCIhJDcvj5UxX9H7grZBHbdJ1cbM/31uUMeU0KViEAkhH02bSYms6qQ1Le7T\ncQOnVf3GZObqXAbZS8UgEkL+9+14TotrR7Cvgt2pRX12xa8ga7fOgBYVg0hImf5HOl0aBnf/AkDV\nyvHEbj+V8XMWBn1sCT0qBpEQseaPzWyNW8jNHc/1ZPyqNGbCj9rPICoGkZDx4riJVNzekuOPC9zV\nVA+lQcXGzFqj/QyiYhAJGZ8vHM85VYNztnNxzk1uzIpdKgZRMYiEBOccS/PT6fmP4O9f2Kdz88Zs\njf+RvPw8zzJIaFAxiISAcbMXQE4Cnc5O9ixDg5PLY7sqMW3JL55lkNCgYhAJAa9lpJNs7YiJCfJx\nqoWYwXE5jflytnZARzsVg0gI+Hb9eDqd5t1mpH1OLtOI6b/+5HUM8ZhfisHM2pvZYjNbamb9DzFd\nMzPLMbMu/hhXJBJs3p7FpoTp9Ol4vtdRSK3VkCV/qhiinc/FYGYxwDCgHdAA6GZm9Q4y3eNAuq9j\nikSS4eMzKLv9TE46oYzXUWjdsCGbbIHXMcRj/lhjSAWWOedWOedygJFA52KmuxX4CPjdD2OKRIyP\n56XT7DjvDlMt7IKmJ5OTsJ6NW3d4HUU85I9iqAZkFnq8puC5/czsROAS59xwwLu9ayIhaOHu8Vx3\njvf7FwASE0qSsKMeY2fp0hjRrGSQxnkOKLzv4ZDlMHjw4P3309LSSEtLC0goEa99t3AFOTHbuDKt\nkddR9qsa05ApixbQvXVzr6PIQWRkZJCRkRGw+ZtzzrcZmLUABjvn2hc8HgA459yQQtOs2HcXqARk\nAb2cc6OLmZ/zNZNIuLj6ueFMXf0Dvz7zltdR9rvwkSf5fddaZj/6nNdR5AiZGc45v22N8cempJlA\nspnVMrM4oCtQ5Be+c65Owa02e/cz3FJcKYhEm0mr02mfHBr7F/ZpflJDVu7SkUnRzOdicM7lAX2B\nCcBCYKRzbpGZ9TazXsW9xNcxRSJBVvYe1idMpm+HNl5HKaJdkxT+jNWRSdHML/sYnHPjgboHPPff\ng0x7vT/GFAl3b0yYRuLOU2lwUmWvoxTR/LRq5MfsYcma36lb/Xiv44gHdOaziEdGzBpPk7KhcTRS\nYTExRpmdDRk7S5uTopWKQcQj87an061ZaO1f2KdGfApTl6sYopWKQcQDP63YQHbCr/Rs18LrKMVq\nULkhCzdqP0O0UjGIeGDYuAmcsLsVCXHBOpXo6Jx3akPW5GiNIVqpGEQ8kL5iPK1rhd7+hX06nNmA\nHaUWkpef73UU8YCKQSTIcnLzWV3yK/q0C839CwAnV6tAzJ4KTFu00uso4gEVg0iQvTdpDnG5lWhe\nr6bXUQ6pYk5D0udqc1I0UjGIBNk709JJSQjdtYV9TkpMYfpKFUM0UjGIBNnMzeO5rEno7l/Yp/GJ\nKSzdoqusRiMVg0gQ/bpuK9tLz+Om9i29jnJYafUb8pvTIavRSMUgEkQvjPmayrvOoXzpUl5HOawL\nz6zH7sTlZGXv8TqKBJmKQSSIxixOp+WJob9/AaBi2VLE7qrJ13OXeR1FgkzFIBIkeXmO5TaeXq1D\nf//CPpVdCl8v0OakaKNiEAmS0d8vISYG2jSu53WUI5ZcNoU5a1QM0UbFIBIkr08ZT90S7TALn489\nP7NGCsu36ZDVaKNiEAmS738fzyUp4bMZCaBVwxQ2xWiNIdqoGESC4Lc/drGlzFT6dGjtdZSj0ur0\nZHJLreX3LVleR5EgUjGIBMFLY76hfHZjTqhQzusoR6VUfCwJO09l3KxFXkeRIFIxiATBpz+lc1bl\n8DhM9UBVY1L4ZrE2J0UTFYNIgDkHi3PGc33L8Nq/sE+9CinMX69iiCYqBpEA+3r2avJLbeSfLZp6\nHeWYNK/dkJU7VQzRRMUgEmCvTEynjmtLiZjw/HFrc3oKW2JVDNEkPN+pImEkY+14OtYNz/0LAC1O\nq0l+3FaWr9nidRQJEr8Ug5m1N7PFZrbUzPoX8/WLzWy+mc01s1lm1sof44qEuj+35bCx9CT6XtjW\n6yjHrERMDKV3NmDcbF2CO1r4XAxmFgMMA9oBDYBuZnbgOf8TnXOnO+eaAD2AV3wdVyQcvDJ2OqVz\nTiK5alWvo/ikelwKU5dpc1K08McaQyqwzDm3yjmXA4wEOheewDm3s9DD0sAmP4wrEvI+nJPOmeXD\n82ikwupXSmHB7yqGaOGPYqgGZBZ6vKbguSLM7BIzWwSMBW7zw7giIc05+GnXeK4+K3z3L+xzdnIK\nmbtVDNGiZLAGcs59BnxmZucC7wB1Dzbt4MGD999PS0sjLS0t0PFE/G76TxvZU3YpV//jbK+j+Kx9\n0xTunv4T+fmOmJjwuQhgpMrIyCAjIyNg8zfnnG8zMGsBDHbOtS94PABwzrkhh3jNL0Cqc+6PYr7m\nfM0kEgquffJ9pvwxilWPf+51FJ855ygxsDIzevzEmXVP8DqOHMDMcM75rbH9sSlpJpBsZrXMLA7o\nCowuPIGZnVzoflOA4kpBJJJ8vWo87eqE//4F2PuLp/yeFNLnaHNSNPC5GJxzeUBfYAKwEBjpnFtk\nZr3NrFfBZJea2QIzmwMMBa70dVyRULYjK5/1iRPo0z789y/sUyuhIT/8qmKIBn7Zx+CcG88B+wyc\nc/8tdP8J4Al/jCUSDv43dj4JVpbTa9bxOorfNKyawverZngdQ4JAZz6LBMCImek0Lh05awsALeum\nsD5XawzRQMUgEgBzd4ynW2pk7F/Y58IzG7Cz9EL25OR7HUUCTMUg4mdzF21jd4XZ9Iiww6yrHVee\nEjkV+Gb+Kq+jSICpGET8bNjYrzgx72xKxyd5HcXvjstLYeJP2pwU6VQMIn6WvmIMbWp38DpGQNRJ\nSmHmShVDpFMxiPhR1s581iWO49Z2Hb2OEhBNqqewbKuKIdKpGET86I1xc0mwsjQ9KdnrKAGRVj+F\nDagYIp2KQcSP3p8xliZlInMzEkC7pqexp/RStu3I8TqKBJCKQcSP5u0cwzXNI3MzEkC5xETidldn\nwuxlXkeRAFIxiPjJjIUb2V1mEdedf57XUQLqeNeQyQu1OSmSqRhE/OT5seOpmdeKhNh4r6ME1Cnl\nUpizVsUQyVQM4pn8fNi2DXbt8jqJf0xcOZaL6kbuZqR9mtVKYcV2FUMkC9oH9YgArFoFr474jZGz\nxrEqbwauzBpcbklK5yTzj2ptefTGVjRsUMLrmEftjy25bCiTzh0dn/Y6SsBd0CiFp+epGCKZikGC\n4uef4Y6np5Kx+zliTv6a5mkX0CvlPOpWbc/u3BymLlnMhwsG0OTVHVwY8wwfPtaRhASvUx+5YZ9P\no2x+LZKrnOh1lIBr2eAU8hIzWbdxFydWLuV1HAkAnz/Bzd/0CW6RZcsWuOM/Kxi5uR9JyXO5N60f\nvZtfS5n4Mn+b1jnHR/PS6fHRLSRmXszs/3uSGtViPUh99FLuGEiNaiUYd/cjXkcJiqR+jRja6k1u\n6NDU6yhCaH6Cm8jf5OXB8y/voHr3exlVrhn9rjqTdfcuol/LPsWWAux9c1/epD2r75tN2ZMXUf/B\nbqzM3BPk5EcvNxcW542hd6vI37+wzwklUvh2sTYnRSoVg/jdd1PzObnLu9y9qh5pnVfzy79/5NH2\n95JQ8si2DVVMrMDC+0ZTreYeznioB1lZob0G+fGkFVjpDXRqkup1lKCpVzGF+etVDJFKxSB+89tv\n0LHXDFq9dw4xZz3HpJs/YEzPd6lWttpRzyu+ZDxzBo7CKi2lxT3/RyhvXfxvxuc0jLuYEjHht9P8\nWJ2dnMLKXT95HUMCRMUgPsvOhgefXMdJd3RnygmXMPSaXizvP4Nzap7t03wT40ox467PWVxmGP2H\nZ/gnbAD8sPVzrm7W2esYQdUp9XS2lZpPXp7XSSQQVAxyzPbsgSEv/kblf/Xj8W0NubbLiay/dwk3\nn9WDGPPPW6tO5RMZ2vo1nv7lOub+vNUv8/SnGQs2kV1uLr3btPY6SlClVK+JxWYzdf5vXkeRAFAx\nyFHLy4NnX19D5Wtv4/719enQKYcVd8/nlSv+76A7ln1xS5sONK94IR1fuDPkNik98+UYauVfQFJ8\ndB22aWYcl9OEL2fP9TqKBICKQY5Yfj4Me3clla67iXt+bUSHtvFkDviZUdcNpXrZ6gEde8wdT7Cp\n3AQe+t/UgI5ztL5a/RmXNbjE6xieqFu2CdN+VTFEIhWDHJZz8PrHK6l8fS/uWHQG7VpWZO2AJYy4\n/kmqlq4alAwVkspwX7OneHReH7ZszQ3KmIfzy+qdbC4/iX9fHD2HqRZ21klNWbJNxRCJ/FIMZtbe\nzBab2VIz61/M168ys/kFt+/MrKE/xpXA+2bOBqrd1Ives8+kVfMqbLhvGSNvfIzjS1cOepYHu1zJ\ncYkV6frkf4M+dnGe+GgiVfLOoGq5il5H8USnZk34I3ZuyG3eE9/5XAxmFgMMA9oBDYBuZlbvgMlW\nAC2dc6cDjwCv+jquBNau7DzaPDCUtA9SqF+nHOvvXcqHNz/McYne/RI0M97s+ixf7XmYpSu3e5Zj\nn9FLP6NjcnQdjVTY2aeeikv6jQXLQ++gAPGNP9YYUoFlzrlVzrkcYCRQ5KfFOfeDc27fu+cH4OgP\nbJegmfrTGqrc04o5Oz8h49pvmNj/SSqXDo2/its1Pp168a256oVnPc2xbsMeNpT/nP6duniaw0sl\nYkpQbndDRk+f73UU8TN/FEM1ILPQ4zUc+hf/DcA4P4wrATB09GRavnsm553Yjg1PTKJl/dO8jvQ3\n717/MHPihjJt/kbPMgz5cCIV80/j1Ko1PMsQCuqUasK3y7SfIdIE9eqqZnY+0AM491DTDR48eP/9\ntLQ00tLSAppL9rrzjREMXXI7jzcbyT2XtfI6zkE1rV2H5ond6P76Yyx93ps1h48Xj6Jj/Ss8GTuU\nNK/ZlC/mf+d1jKiTkZFBRkZGwObv89VVzawFMNg5177g8QDAOeeGHDBdI+BjoL1z7pdDzE9XV/XA\nHf97hxd+HsiIi8ZxxT9C/9iAFRs2kPxcfb66dB6tzwzuX+1rN+ym+nNVWXLbQk49IfIvs30oo2fN\n5tK3e5Dz/I9eR4lqoXh11ZlAspnVMrM4oCswuvAEZlaTvaVwzaFKQbwxeNQnvLDoHkZ1/CosSgGg\nTpUqtEy6kd7vPhr0sR8ZkU7lvEZRXwoAbU9PIbfscjLXZ3sdRfzI52JwzuUBfYEJwEJgpHNukZn1\nNrNeBZM9AFQEXjKzuWY2w9dxxT8+mT6Th+b25uWWY7gsLfT2JxzKm73uZkXCR0yYuSKo4368+AO6\n1L0yqGOGqoTYeJJ2n8Jn3+tKq5FEH9QTxX75bQP1nmlGzxOf5+U7wvPs3dYPD+bXLStZ8cybQRlv\n8S87qf9aNVbds5gaFaoEZcxQ1+iBnpwU24zRD97kdZSoFYqbkiQM5eXncfYzXWmUd33YlgLAmzfd\nycq4MYybuSQo4w0a8SnVaaFSKOSsmqnM26iNAJFExRClrhn+DDuy8pny0ANeR/FJjcrlaJ14FzeP\nGhzwsZyDMevepOcZPQI+Vjj5Z7NU1sVM1xnQEUTFEIXGz5vPyMwnGNXtLUonhf+Hy7x5862sjpnM\n6OmB/eCYMd+tZle5Odx98cUBHSfctEpJIb/MKhYs2+Z1FPETFUOUycnL5Yr3r+OSxCe56NyTvI7j\nF9Uql6Z9mXvo8+GggI4z+LO3ObPUlSTGHdlHlEaLuJKxVNjdmA+/m+11FPETFUOU6f36i+TtqMCI\n/t29juJXb/a5mXU2nQ+nBuaX07Ztjrl5b/HgxdcFZP7h7rRyqUxaNt3rGOInKoYosuy3dbz168MM\nv+gl4uP9dgBDSDi+YikurnAft336YEDmP+jtr0lKSKDD6c0CMv9wl5acyqKt2gEdKVQMUeTiF/9N\ng129ubbDgRe/jQyv9+nJ7yzg7cnf+3W+zsGbC4fRvd6tmEVWofrL5WensiVxBjk5XicRf1AxRIlP\nZvzA0uxv+bzffV5HCZiK5eK5rPKD/HuMf4+0+mjiSraV/45Hr/yXX+cbSRrVqE1M7G4mzVzrdRTx\nAxVDFHDOcdPH93BxmYeoXT3R6zgB9Wqfa9mSv5r/pk/22zzv/XQ451foTtlSSX6bZ6QxM07IT+XT\nGdqcFAlUDFHgmbFfsCV7M2/dGVk7nItTtnQsV1cfRP/0B8jP9/3A+pk//ckvZV/n+av7+iFdZGtc\nuTnfrZrmdQzxAxVDhMvJy+XBKQO4KXkIZcuE/zkLR+LlPt3YxWb+8166z/O66Y1hNEq4iPon1vZD\nssj2z6bnsTznW69jiB+oGCJcv3ffhKzjeeamDl5HCZqE+BLc0+whhsx4gOzsY19rmLNgB3PjnufV\n7gP9mC5yXXFOc3aX/5FfMnd6HUV8pGKIYHtyc3j550e4t/ljxMZG19E0/+nahfjEPLo/NeKY53HN\n8KdoVLoNzWrX9WOyyFU6PpHyuxvy9kTtZwh3KoYI1v/9d4jdnszAq8/2OkrQxVgMb132Mh9uu4up\nczcd9es/mbiWRWVfYETPxwKQLnI1Kn8u6Yu0OSncqRgiVE5eLsMXPEa/Zg8SE6X/ypc0S6VV5X9x\n8fDbyc098tdlZzu6j7yVS2vcwmkn1gpcwAjUseF5LNyhj/oMd1H6KyPy3T9qBDFZ1Xng2pZeR/HU\nZ7c9xO4Kc+lw/+tH/JorHvoAq7SEd268P4DJItPVLc9hR/lpbNp8FE0sIUfFEIFy8/J4Yd4j3N74\nAUpEx4FIB1U6PomJN37C1zaAe4cf/lDKF0cuYUz+rXx0zf9IiI0PQsLIcmL5SiTmVue9r+d7HUV8\noGKIQIM/+gC3sxIPXdfK6yghoUVyPYa3eYchKy9h0MtzDzrd+1+u4fbpnRiQ+hhtG6QGMWFkqZd4\nHqPnaT9DOFMxRJh8l8+zsx/hlgYPRN2RSIfSq1V7hrZ9mUdWtqP1ne+xadNfh7FmZ0Ofx2ZwzeTz\n6HXGjTza5QYPk4a/jvVbMXvL117HEB/oM58jzEMffcRjU55g29PTiYtTMRxoyrI5XPrW9fy5uSQ1\ndl1MfEwpVuR+h9X4gWdaD6PP+Zd7HTHsrdmykRpPJrPhrk0cXynW6zhRwd+f+axiiCD5Lp9y/Ztw\nXc1HeKFvJ6/jhKx8l8+Hc9P5eNYUsnN3c/5pjel1zmUkxelaSP5S9p4m9E8Zxn3XnuN1lKigYpCD\nGvLZ5zwwaTDbn5wTcZ+3IOHl/MfuYfvmJGY9FdhP1ZO9/F0M2scQIZxzPDb1Ya4/+UGVgniua+oF\nLNj1ldcx5Bj5pRjMrL2ZLTazpWbWv5iv1zWz780s28zu8seYUtRzY8axa89unu3d2esoIlzT8jz2\nVJjH/MXbvI4ix8DnYjCzGGAY0A5oAHQzswM/IuwP4FbgSV/Hk79zzvHQlIe4utb9lErQSqB4LzGu\nFFVymvPy+Ayvo8gx8MdvkVRgmXNulXMuBxgJFPmz1Tm3yTk3G9DpkAHw0viJ7MjZxgs3XeZ1FJH9\nWtfqyNjlX3odQ46BP4qhGpBZ6PGaguckCJxzPDDpP1xV/X6SEqP8NGcJKXdc2InMUl+wfUe+11Hk\nKJX0OkBxBg8evP9+WloaaWlpnmUJda9OzGBb7kZeuuVKr6OIFHFm7VMoFVOel0fP5u6rmnkdJ6Jk\nZGSQkZERsPn7fLiqmbUABjvn2hc8HgA459yQYqYdBGx3zj1ziPnpcNWjULnf+bSt0oP37r7W6ygi\nf9P68f78sSGOec8+7HWUiBaKh6vOBJLNrJaZxQFdgdGHmF7HUvrJG19/w5b8TF6+5Sqvo4gUq3da\nJxbkjiYvz+skcjR8LgbnXB7QF5gALARGOucWmVlvM+sFYGZVzCwTuBO4z8xWm1lpX8eOdgPHPcyl\nVe6lTFJIbhEU4dLUs6DMOj6Z/KvXUeQo6MznMPV2xlSu/+Jq/hi8lHJldD0aCV1n/qc38TtrM3XI\nAK+jRKxQ3JQkQeac484vB3JFlUEqBQl5t7fqxoydI47qU/TEWyqGMPT05+PZnreJ1269xusoIod1\n1TnnYaU38dbYn72OIkdIxRBm8vLzGfTtQG469VESS+m8BQl9JWJKcHbZrrwweYTXUeQIqRjCTP93\nRuFy43nmxku8jiJyxAZ0uIqf7D12ZOlkt3CgYggjWdm7eWHBA9zX/DFKltRRvxI+2jVqSlLJsvzn\nHX2yWzhQMYSRf70wlDK763Fvt9ZeRxE5KmbGv+r25o35//U6ihwBFUOYWLBqPaP/eIK3uj2LaWVB\nwtCjXf9kZyT6AAAKdklEQVTFlgpfM2nGb15HkcNQMYSJS18aSFN60vGsU7yOInJMKiaVpXH8pQz4\n4DWvo8hhqBjCwOsTvmd5/ld8etf9XkcR8ckzl9/BrJhhrFq3y+socggqhhC3bWc2fdJv4JaTn6PG\n8WW8jiPik7T6KdQskUrv4W96HUUOQcUQ4jo+8TAV8uoxtJc+hEciw5OdB/BV1lNs/lOnQocqFUMI\nG/XNXKbuepUxfV4kJkZ7nCUyXN7ibCrH1qTH0Le8jiIHoWIIURu27ODaz7vRq9azND3lBK/jiPjV\ny5cN4Ysdg1iRmeV1FCmGrq4aopL7dccowbKn3vA6ikhA1L3/SsruasjMp3VQha90ddUocN3zr5OZ\nP5Op973gdRSRgBl5w/8xO/Y5PpuywusocgAVQ4h5+pPJvL32Xj7t+gnHV0jyOo5IwDQ5qQ5da/Tn\n6g9uYFe2rqEUSlQMIWTM9MXcPb0rTzYfQYfUel7HEQm4t2+6i9ikLDo/8pLXUaQQFUOISJ+1jM4f\nteGGWk/w7y6tvI4jEhQlS5RgbM93+Tr3IZ4YMc3rOFJAxRACxk5fQsdRrbi25mBeuaW713FEguqs\nuqfwxDlvMHDO5UyatcbrOIKKwXPPfvINF33SkutrP8wbt/b0Oo6IJ/7d6SK61rqLtu+3YuqP672O\nE/V0uKpHcvPyufSJF/jiz0d5ovl79OvSxutIIp7757OP8uXqdxh95VgubFHH6zhhw9+Hq6oYPPDt\nj6u45LWbyCm5hc+7v8v5pyd7HUkkZPQY/iJv//oIgxq+x4PXaH/bkVAxhLGV67fSbdjTTM97kbZl\n7+Szfv1JiIv1OpZIyBk2diJ3TOnOKTldGHf3I5x0QjmvI4W0kDzBzczam9liM1tqZv0PMs3zZrbM\nzOaZWWN/jBsuxs9YTpP+/agztA6bclYyo+c8xt97v0pB5CD6driAFf1+IiZ+J3WeO4V2Dz3Jr+v/\n9DpW1PB5jcHMYoClQGtgHTAT6OqcW1xomguBvs65jmbWHBjqnGtxkPmF/RpD1q5c3ps0hw9nTWbq\n5o/YnZDJGXFXMezq20mtW8vreCJhZdyshdwy6hFWlhxH7T2dubJRF267qDUnHFfa62ghI+Q2JZlZ\nC2CQc+7CgscDAOecG1JompeByc65UQWPFwFpzrkNxcwvLIohe3cei1Zv5OfM9Sxbv54f1/zCok0/\nsy5nIdsS55Owuxb1Ev7BNakX07fj+cSVLOl1ZJGwtjjzd+4d+T5T1n3J5sTpJO2sR624Mzi9aiMa\nVKtNk9q1SK1bk+PKJGFR9vm3/i4Gf/y2qgZkFnq8Bkg9zDRrC577WzEA1L/nJhwO2FsQzjn2/Wcv\nd8jnCh4V/O/e5/dPc+Br9n/9768DyHV7yGEnOZZFnu0kr0QW+SWycHHbiNldkbicqpR2J1A14SQa\nVm1Ar5Mvp3OLRtSpWulIl5+IHIF6NY7nk7vvAO7g9y1ZfPTdPCYums3stfNIX/k5239YSU5iJsTk\nErOnPCVzyxObX46SlKIkcZQouJW0OGJj4oihBGB/lYizgs9TN4y9z+37/30ftG6FvsaB00SQkPwz\nNmZe5v5/rKqn1KVKcl1iCv4BzQr9k5kRY3/9wxqFHttf/4hFpin0mn2v2v+4YJrCj+NjS1IhMYkK\npZOoWDqJCmUSqVQ2iZrHlyc+NiQXn0jEO75CErd0OodbOp1T5HnnYPPW3WRu/JM1f/zJ2j/+JCt7\nN9k5e9iVs4fsPXvYnbuH7Jw95Obn7v+jMX/fH4uF/rjMd3/9YQp//Vm5979//XHphd+XL+X3X5YF\nbP7++M22FqhZ6HH1gucOnKbGYabZb8GEMX6IJSLRxgyOKx/PceWr0PiUKl7HCRp/bzrzx1FJM4Fk\nM6tlZnFAV2D0AdOMBq6F/fsk/ixu/4KIiHjP5zUG51yemfUFJrC3aF53zi0ys957v+xecc6NNbMO\nZrYcyAJ6+DquiIgEhk5wExEJcyF5gpuIiEQOFYOIiBShYhARkSJUDCIiUoSKQUREilAxiIhIESoG\nEREpQsUgIiJFqBhERKQIFYOIiBShYhARkSJUDCIiUoSKQUREilAxiIhIESoGEREpQsUgIiJFqBhE\nRKQIFYOIiBShYhARkSJUDCIiUoSKQUREilAxiIhIET4Vg5lVMLMJZrbEzNLNrNxBpnvdzDaY2Y++\njCciIoHn6xrDAGCic64uMAkYeJDp/ge083EsOUoZGRleR4goWp7+peUZunwths7AWwX33wIuKW4i\n59x3wBYfx5KjpB88/9Ly9C8tz9DlazEc75zbAOCc+w043vdIIiLipZKHm8DMvgKqFH4KcMD9xUzu\n/JRLREQ8Ys4d++9yM1sEpDnnNphZVWCyc+60g0xbC/jCOdfoMPNUuYiIHCXnnPlrXoddYziM0cB1\nwBCgO/D5Iaa1gtsh+fObExGRo+frPoYhQBszWwK0Bh4HMLMTzOzLfROZ2fvA98CpZrbazHr4OK6I\niASIT5uSREQk8oTMmc9m1t7MFpvZUjPr73WecGFmK81svpnNNbMZBc8d9MRDMxtoZsvMbJGZtfUu\nufeKO/HyWJadmTU1sx8L3rvPBfv7CBUHWZ6DzGyNmc0puLUv9DUtz4Mws+pmNsnMFprZT2Z2W8Hz\nwXl/Ouc8v7G3oJYDtYBYYB5Qz+tc4XADVgAVDnhuCHBPwf3+wOMF9+sDc9m7b+mkgmVuXn8PHi67\nc4HGwI++LDtgOtCs4P5YoJ3X31sILc9BwF3FTHualuchl2VVoHHB/dLAEqBesN6fobLGkAosc86t\ncs7lACPZe/KcHJ7x9zW/g514eDEw0jmX65xbCSxj77KPSq74Ey+PatkVHI1Xxjk3s2C6tznIiZ6R\n7iDLE4o/6KQzWp4H5Zz7zTk3r+D+DmARUJ0gvT9DpRiqAZmFHq8peE4OzwFfmdlMM7uh4LkqrvgT\nDw9czmvRcj7QwU7aPNiyq8be9+s+eu/+XV8zm2dmrxXa9KHleYTM7CT2ron9wNH/bB/T8gyVYpBj\nd45zrinQAehjZufx9xMNdYTBsdOy881LQB3nXGPgN+Bpj/OEFTMrDXwE3F6w5hCUn+1QKYa1QM1C\nj6sXPCeH4ZxbX/D/G4HP2LtpaIOZVQEoWJX8vWDytUCNQi/Xcv67o112WqaH4Jzb6Ao2bgOv8tem\nSy3PwzCzkuwthXecc/vOEQvK+zNUimEmkGxmtcwsDujK3pPn5BDMLLHgLwrMLAloC/zEXyceQtET\nD0cDXc0szsxqA8nAjKCGDj0Hnnh5VMuuYHV+q5mlmpkB13LoEz0jXZHlWfDLa58uwIKC+1qeh/cG\n8LNzbmih54Lz/vR673uhvfDt2bvnfRkwwOs84XADarP3CK657C2EAQXPVwQmFizPCUD5Qq8ZyN4j\nFhYBbb3+Hjxefu8D64DdwGqgB1DhaJcdcEbB8l8GDPX6+wqx5fk28GPB+/Qz9m4j1/I8/LI8B8gr\n9PM9p+B35FH/bB/L8tQJbiIiUkSobEoSEZEQoWIQEZEiVAwiIlKEikFERIpQMYiISBEqBhERKULF\nICIiRagYRESkiP8H5j87XpiRji0AAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.plot(cvx.conv(c, x).value)\n", + "plt.plot(np.convolve(c, x0))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 3 ms, sys: 2.27 ms, total: 5.28 ms\n", + "Wall time: 4.59 ms\n" + ] + }, + { + "data": { + "text/plain": [ + "array([ -1.13686838e-16, -1.02318154e-16, -1.47792889e-16, ...,\n", + " -2.95585778e-16, -2.50111043e-16, -3.18323146e-16])" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from scipy.signal import fftconvolve\n", + "%time fftconvolve(c, x0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "x = cvx.Variable(n)\n", + "f = cvx.conv(c, x)\n", + "prob = cvx.Problem(cvx.Minimize(0), [f == 0])\n", + "A = prob.get_problem_data(cvx.SCS)[\"A\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ValueError", + "evalue": "dimension mismatch", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mu'time A*x0'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mmagic\u001b[0;34m(self, arg_s)\u001b[0m\n\u001b[1;32m 2161\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marg_s\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2162\u001b[0m \u001b[0mmagic_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprefilter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mESC_MAGIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2163\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2165\u001b[0m \u001b[0;31m#-------------------------------------------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line)\u001b[0m\n\u001b[1;32m 2082\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2083\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2084\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2085\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2086\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/IPython/core/magic.pyc\u001b[0m in \u001b[0;36m\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/IPython/core/magics/execution.pyc\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[1;32m 1171\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'eval'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1172\u001b[0m \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1173\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1174\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1175\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/site-packages/scipy/sparse/base.pyc\u001b[0m in \u001b[0;36m__mul__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m 369\u001b[0m \u001b[0;31m# dense row or column vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 370\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 371\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'dimension mismatch'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 372\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 373\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mul_vector\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: dimension mismatch" + ] + } + ], + "source": [ + "%time A*x0" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 9a25d7d7f2..295f6eb386 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -34,11 +34,12 @@ - 2.7.8 + 4.3.6 + 2.22.2 2.6.0 2.2.1 1.13 - 2.0-m10 + 2.0.1 1.8 4.1.0 @@ -86,6 +87,12 @@ ${project.groupId} zeppelin-zengine ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + @@ -98,6 +105,45 @@ slf4j-log4j12 + + org.glassfish.jersey.core + jersey-client + ${jersey.version} + + + javax.annotation + javax.annotation-api + + + + + org.glassfish.jersey.containers + jersey-container-servlet-core + ${jersey.version} + + + org.glassfish.jersey.media + jersey-media-json-jackson + ${jersey.version} + + + com.fasterxml.jackson.core + jackson-annotations + + + + + org.glassfish.jersey.core + jersey-server + ${jersey.version} + + + + javax.ws.rs + javax.ws.rs-api + ${javax.ws.rsapi.version} + + commons-collections commons-collections @@ -131,54 +177,11 @@ ${jna.version} + - org.apache.cxf - cxf-rt-frontend-jaxrs - ${cxf.version} - - - javax.ws.rs - javax.ws.rs-api - - - - - - org.apache.cxf - cxf-rt-transports-http - ${cxf.version} - - - - org.apache.cxf - cxf-rt-transports-http-jetty - ${cxf.version} - - - org.eclipse.jetty - jetty-server - - - org.eclipse.jetty - jetty-security - - - org.slf4j - slf4j-api - - - - - - org.apache.cxf - cxf-api - ${cxf.version} - - - org.eclipse.jetty - * - - + com.fasterxml.jackson.core + jackson-annotations + 2.5.4 @@ -292,18 +295,6 @@ ${quartz.scheduler.version} - - com.sun.jersey - jersey-servlet - ${jersey.servlet.version} - - - - javax.ws.rs - javax.ws.rs-api - ${javax.ws.rsapi.version} - - org.scala-lang scala-library diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ActiveDirectoryGroupRealm.java b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ActiveDirectoryGroupRealm.java index d40a64378d..4f3626cbc5 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ActiveDirectoryGroupRealm.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/ActiveDirectoryGroupRealm.java @@ -297,6 +297,16 @@ public class ActiveDirectoryGroupRealm extends AbstractLdapRealm { return userNameList; } + public Map getListRoles() { + Map roles = new HashMap<>(); + Iterator it = this.groupRolesMap.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry) it.next(); + roles.put((String) pair.getValue(), "*"); + } + return roles; + } + private Set getRoleNamesForUser(String username, LdapContext ldapContext) throws NamingException { Set roleNames = new LinkedHashSet<>(); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapRealm.java b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapRealm.java index 3381a516c5..97c223c2fc 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapRealm.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/realm/LdapRealm.java @@ -109,6 +109,9 @@ import javax.naming.ldap.PagedResultsControl; *

    # optional mapping from physical groups to logical application roles * ldapRealm.rolesByGroup = \ LDN_USERS: user_role,\ NYK_USERS: user_role,\ * HKG_USERS: user_role,\ GLOBAL_ADMIN: admin_role,\ DEMOS: self-install_role + * + *

    # optional list of roles that are allowed to authenticate + * ldapRealm.allowedRolesForAuthentication = admin_role,user_role * *

    ldapRealm.permissionsByRole=\ user_role = *:ToDoItemsJdo:*:*,\ * *:ToDoItem:*:*; \ self-install_role = *:ToDoItemsFixturesService:install:* ; @@ -176,6 +179,7 @@ public class LdapRealm extends JndiLdapRealm { private String memberAttributeValueSuffix = ""; private final Map rolesByGroup = new LinkedHashMap(); + private final List allowedRolesForAuthentication = new ArrayList(); private final Map> permissionsByRole = new LinkedHashMap>(); @@ -202,6 +206,29 @@ public class LdapRealm extends JndiLdapRealm { } } + /** + * This overrides the implementation of queryForAuthenticationInfo inside JndiLdapRealm. + * In addition to calling the super method for authentication it also tries to validate + * if this user has atleast one of the allowed roles for authentication. In case the property + * allowedRolesForAuthentication is empty this check always returns true. + * + * @param token the submitted authentication token that triggered the authentication attempt. + * @param ldapContextFactory factory used to retrieve LDAP connections. + * @return AuthenticationInfo instance representing the authenticated user's information. + * @throws NamingException if any LDAP errors occur. + */ + @Override + protected AuthenticationInfo queryForAuthenticationInfo(AuthenticationToken token, + LdapContextFactory ldapContextFactory) + throws NamingException { + AuthenticationInfo info = super.queryForAuthenticationInfo(token, ldapContextFactory); + // Credentials were verified. Verify that the principal has all allowedRulesForAuthentication + if (!hasAllowedAuthenticationRules(info.getPrincipals(), ldapContextFactory)) { + throw new NamingException("Principal does not have any of the allowedRolesForAuthentication"); + } + return info; + } + /** * Get groups from LDAP. * @@ -231,6 +258,23 @@ public class LdapRealm extends JndiLdapRealm { return simpleAuthorizationInfo; } + private boolean hasAllowedAuthenticationRules(PrincipalCollection principals, + final LdapContextFactory ldapContextFactory) + throws NamingException { + boolean allowed = allowedRolesForAuthentication.isEmpty(); + if (!allowed) { + Set roles = getRoles(principals, ldapContextFactory); + for (String allowedRole: allowedRolesForAuthentication) { + if (roles.contains(allowedRole)) { + log.debug("Allowed role for user [" + allowedRole + "] found."); + allowed = true; + break; + } + } + } + return allowed; + } + private Set getRoles(PrincipalCollection principals, final LdapContextFactory ldapContextFactory) throws NamingException { @@ -524,6 +568,10 @@ public class LdapRealm extends JndiLdapRealm { this.memberAttributeValueSuffix = suffix; } + public void setAllowedRolesForAuthentication(List allowedRolesForAuthencation) { + this.allowedRolesForAuthentication.addAll(allowedRolesForAuthencation); + } + public void setRolesByGroup(Map rolesByGroup) { this.rolesByGroup.putAll(rolesByGroup); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 8ca04762d3..9c511d46fe 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -588,17 +588,17 @@ public class NotebookRestApi { throws IOException, IllegalArgumentException { LOG.info("run note jobs {} ", noteId); Note note = notebook.getNote(noteId); + AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); checkIfNoteIsNotNull(note); checkIfUserCanWrite(noteId, "Insufficient privileges you cannot run job for this note"); try { - note.runAll(); + note.runAll(subject); } catch (Exception ex) { LOG.error("Exception from run", ex); return new JsonResponse<>(Status.PRECONDITION_FAILED, ex.getMessage() + "- Not selected or Invalid Interpreter bind").build(); } - return new JsonResponse<>(Status.OK).build(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java index 9d9f5510a0..42edd235ed 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/ZeppelinRestApi.java @@ -17,12 +17,18 @@ package org.apache.zeppelin.rest; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; import org.apache.zeppelin.annotation.ZeppelinApi; import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.util.Util; +import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; +import javax.ws.rs.PUT; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; /** @@ -52,4 +58,25 @@ public class ZeppelinRestApi { public Response getVersion() { return new JsonResponse<>(Response.Status.OK, "Zeppelin version", Util.getVersion()).build(); } + + /** + * Set the log level for root logger + * @param request + * @param logLevel new log level for Rootlogger + * @return + */ + @PUT + @Path("log/level/{logLevel}") + public Response changeRootLogLevel(@Context HttpServletRequest request, + @PathParam("logLevel") String logLevel) { + Level level = Level.toLevel(logLevel); + if (logLevel.toLowerCase().equalsIgnoreCase(level.toString().toLowerCase())) { + Logger.getRootLogger().setLevel(level); + return new JsonResponse<>(Response.Status.OK).build(); + } else { + return new JsonResponse<>(Response.Status.NOT_ACCEPTABLE, + "Please check LOG level specified. Valid values: DEBUG, ERROR, FATAL, " + + "INFO, TRACE, WARN").build(); + } + } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 2b67dfdc3a..826ae5f863 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -27,7 +27,6 @@ import javax.servlet.DispatcherType; import javax.ws.rs.core.Application; import org.apache.commons.lang.StringUtils; -import org.apache.cxf.jaxrs.servlet.CXFNonSpringJaxrsServlet; import org.apache.shiro.web.env.EnvironmentLoaderListener; import org.apache.shiro.web.servlet.ShiroFilter; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -286,7 +285,7 @@ public class ZeppelinServer extends Application { final ServletHolder servletHolder = new ServletHolder(notebookWsServer); servletHolder.setInitParameter("maxTextMessageSize", maxTextMessageSize); - final ServletContextHandler cxfContext = new ServletContextHandler( + final ServletContextHandler context = new ServletContextHandler( ServletContextHandler.SESSIONS); webapp.addServlet(servletHolder, "/ws/*"); @@ -316,13 +315,15 @@ public class ZeppelinServer extends Application { private static void setupRestApiContextHandler(WebAppContext webapp, ZeppelinConfiguration conf) { - final ServletHolder cxfServletHolder = new ServletHolder(new CXFNonSpringJaxrsServlet()); - cxfServletHolder.setInitParameter("javax.ws.rs.Application", ZeppelinServer.class.getName()); - cxfServletHolder.setName("rest"); - cxfServletHolder.setForcedPath("rest"); + final ServletHolder servletHolder = new ServletHolder( + new org.glassfish.jersey.servlet.ServletContainer()); + + servletHolder.setInitParameter("javax.ws.rs.Application", ZeppelinServer.class.getName()); + servletHolder.setName("rest"); + servletHolder.setForcedPath("rest"); webapp.setSessionHandler(new SessionHandler()); - webapp.addServlet(cxfServletHolder, "/api/*"); + webapp.addServlet(servletHolder, "/api/*"); String shiroIniPath = conf.getShiroPath(); if (!StringUtils.isBlank(shiroIniPath)) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 87d0009473..5588be0218 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -19,15 +19,9 @@ package org.apache.zeppelin.socket; import java.io.IOException; import java.net.URISyntaxException; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.regex.Matcher; @@ -1696,6 +1690,23 @@ public class NotebookServer extends WebSocketServlet p.setErrorMessage((String) fromMessage.get("errorMessage")); p.setStatusWithoutNotification(status); + // Spell uses ISO 8601 formatted string generated from moment + String dateStarted = (String) fromMessage.get("dateStarted"); + String dateFinished = (String) fromMessage.get("dateFinished"); + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX"); + + try { + p.setDateStarted(df.parse(dateStarted)); + } catch (ParseException e) { + LOG.error("Failed parse dateStarted", e); + } + + try { + p.setDateFinished(df.parse(dateFinished)); + } catch (ParseException e) { + LOG.error("Failed parse dateFinished", e); + } + addNewParagraphIfLastParagraphIsExecuted(note, p); if (!persistNoteWithAuthInfo(conn, note, p)) { return; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java index dcb5a1f339..19eb980c61 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/utils/SecurityUtils.java @@ -34,6 +34,7 @@ import org.apache.shiro.subject.Subject; import org.apache.shiro.util.ThreadContext; import org.apache.shiro.web.mgt.DefaultWebSecurityManager; import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.realm.ActiveDirectoryGroupRealm; import org.apache.zeppelin.realm.LdapRealm; import org.mortbay.log.Log; import org.slf4j.Logger; @@ -133,6 +134,9 @@ public class SecurityUtils { } else if (name.equals("org.apache.zeppelin.realm.LdapRealm")) { allRoles = ((LdapRealm) realm).getListRoles(); break; + } else if (name.equals("org.apache.zeppelin.realm.ActiveDirectoryGroupRealm")) { + allRoles = ((ActiveDirectoryGroupRealm) realm).getListRoles(); + break; } } if (allRoles != null) { diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java index add23ac0fb..fa5bed994b 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/ParagraphActionsIT.java @@ -253,7 +253,6 @@ public class ParagraphActionsIT extends AbstractZeppelinIT { } catch (Exception e) { handleException("Exception in ParagraphActionsIT while testDisableParagraphRunButton ", e); } - } @Test @@ -594,6 +593,7 @@ public class ParagraphActionsIT extends AbstractZeppelinIT { CoreMatchers.equalTo("Howdy ")); Select dropDownMenu = new Select(driver.findElement(By.xpath("(" + (getParagraphXPath(1) + "//select)[1]")))); + dropDownMenu.selectByVisibleText("Alice"); collector.checkThat("After selection in drop down menu, output should display the newly selected option", driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@class, 'text plainTextContent')]")).getText(), diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java index 2cc6a6c8a2..0aa03541ff 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java @@ -32,6 +32,8 @@ import org.openqa.selenium.WebElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + public class SparkParagraphIT extends AbstractZeppelinIT { private static final Logger LOG = LoggerFactory.getLogger(SparkParagraphIT.class); @@ -182,10 +184,16 @@ public class SparkParagraphIT extends AbstractZeppelinIT { ); } - WebElement paragraph1Result = driver.findElement(By.xpath( - getParagraphXPath(1) + "//div[contains(@id,\"_graph\")]/div/div/div/div/div[1]")); + // Age, Job, Marital, Education, Balance + List tableHeaders = driver.findElements(By.cssSelector("span.ui-grid-header-cell-label")); + String headerNames = ""; + + for(WebElement header : tableHeaders) { + headerNames += header.getText().toString() + "|"; + } + collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ", - paragraph1Result.getText().toString(), CoreMatchers.equalTo("age\n▼\njob\n▼\nmarital\n▼\neducation\n▼\nbalance\n▼\n30 unemployed married primary 1787")); + headerNames, CoreMatchers.equalTo("Age|Job|Marital|Education|Balance|")); } catch (Exception e) { handleException("Exception in SparkParagraphIT while testSqlSpark", e); } diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index 7ea2774c9d..00b82fdd6f 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -429,13 +429,19 @@ public abstract class AbstractTestRestApi { LOG.info("{} - {}", postMethod.getStatusCode(), postMethod.getStatusText()); Pattern pattern = Pattern.compile("JSESSIONID=([a-zA-Z0-9-]*)"); Header[] setCookieHeaders = postMethod.getResponseHeaders("Set-Cookie"); + String jsessionId = null; for (Header setCookie : setCookieHeaders) { java.util.regex.Matcher matcher = pattern.matcher(setCookie.toString()); if (matcher.find()) { - return matcher.group(1); + jsessionId = matcher.group(1); } } - return StringUtils.EMPTY; + + if (jsessionId != null) { + return jsessionId; + } else { + return StringUtils.EMPTY; + } } protected static boolean userAndPasswordAreNotBlank(String user, String pwd) { diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java index 4ee5a0078a..5093cb838f 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java @@ -167,7 +167,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi { assertEquals("paragraph col width check failed", 9.0, p.getConfig().get("colWidth")); assertTrue("paragraph show title check failed", ((boolean) p.getConfig().get("title"))); Map graph = ((List)p.getConfig().get("results")).get(0); - String mode = graph.get("mode").toString(); + String mode = ((Map)graph.get("graph")).get("mode").toString(); assertEquals("paragraph graph mode check failed", "pieChart", mode); } } diff --git a/zeppelin-web/.eslintrc b/zeppelin-web/.eslintrc index 1fe3fa5e7d..6dca5c8982 100644 --- a/zeppelin-web/.eslintrc +++ b/zeppelin-web/.eslintrc @@ -58,6 +58,7 @@ "no-undef": 2, "no-unused-vars": [2, { "vars": "local", "args": "none" }], "strict": [2, "global"], - "max-len": [2, {"code": 120, "ignoreComments": true, "ignoreRegExpLiterals": true}] + "max-len": [2, {"code": 120, "ignoreComments": true, "ignoreRegExpLiterals": true}], + "linebreak-style": 0 } } diff --git a/zeppelin-web/bower.json b/zeppelin-web/bower.json index 30fc5f0268..8e9a1e590c 100644 --- a/zeppelin-web/bower.json +++ b/zeppelin-web/bower.json @@ -30,8 +30,6 @@ "ngtoast": "~2.0.0", "ng-focus-if": "~1.0.2", "bootstrap3-dialog": "bootstrap-dialog#~1.34.7", - "handsontable": "~0.24.2", - "moment-duration-format": "^1.3.0", "select2": "^4.0.3", "MathJax": "2.7.0", "ngclipboard": "^1.1.1" diff --git a/zeppelin-web/karma.conf.js b/zeppelin-web/karma.conf.js index f47741aee5..b163b56f38 100644 --- a/zeppelin-web/karma.conf.js +++ b/zeppelin-web/karma.conf.js @@ -81,11 +81,6 @@ module.exports = function(config) { 'bower_components/ngtoast/dist/ngToast.js', 'bower_components/ng-focus-if/focusIf.js', 'bower_components/bootstrap3-dialog/dist/js/bootstrap-dialog.min.js', - 'bower_components/zeroclipboard/dist/ZeroClipboard.js', - 'bower_components/moment/moment.js', - 'bower_components/pikaday/pikaday.js', - 'bower_components/handsontable/dist/handsontable.js', - 'bower_components/moment-duration-format/lib/moment-duration-format.js', 'bower_components/select2/dist/js/select2.js', 'bower_components/MathJax/MathJax.js', 'bower_components/clipboard/dist/clipboard.js', diff --git a/zeppelin-web/package.json b/zeppelin-web/package.json index a992a0f79d..ec62003793 100644 --- a/zeppelin-web/package.json +++ b/zeppelin-web/package.json @@ -9,8 +9,8 @@ "clean": "rimraf dist && rimraf .tmp", "postinstall": "bower install --silent", "prebuild": "npm-run-all clean lint:once", - "build:dist": "grunt pre-webpack-dist && webpack && grunt post-webpack-dist", - "build:ci": "grunt pre-webpack-ci && webpack && grunt post-webpack-dist", + "build:dist": "npm-run-all prebuild && grunt pre-webpack-dist && webpack && grunt post-webpack-dist", + "build:ci": "npm-run-all prebuild && grunt pre-webpack-ci && webpack && grunt post-webpack-dist", "lint:watch": "esw --watch src", "lint:once": "eslint src", "predev": "grunt pre-webpack-dev", @@ -18,14 +18,18 @@ "dev:helium": "HELIUM_BUNDLE_DEV=true webpack-dev-server --hot", "dev:watch": "grunt watch-webpack-dev", "dev": "npm-run-all --parallel dev:server lint:watch dev:watch", - "pretest": "npm install karma-phantomjs-launcher babel-polyfill", "test": "karma start karma.conf.js" }, "dependencies": { + "angular-viewport-watch": "github:shahata/angular-viewport-watch", "github-markdown-css": "2.6.0", + "angular-ui-grid": "^4.0.4", "grunt-angular-templates": "^0.5.7", "grunt-dom-munger": "^3.4.0", - "headroom.js": "^0.9.3" + "headroom.js": "^0.9.3", + "moment": "^2.18.1", + "moment-duration-format": "^1.3.0", + "scrollmonitor": "^1.2.3" }, "devDependencies": { "autoprefixer": "^6.5.4", @@ -73,6 +77,7 @@ "karma-coverage": "^1.1.1", "karma-jasmine": "~1.0.2", "karma-sourcemap-loader": "^0.3.7", + "karma-phantomjs-launcher": "^1.0.4", "karma-webpack": "^1.8.1", "load-grunt-tasks": "^0.4.0", "ng-annotate-loader": "^0.2.0", diff --git a/zeppelin-web/src/WEB-INF/web.xml b/zeppelin-web/src/WEB-INF/web.xml index e2514c9d4c..4356b7c080 100644 --- a/zeppelin-web/src/WEB-INF/web.xml +++ b/zeppelin-web/src/WEB-INF/web.xml @@ -23,11 +23,12 @@ zeppelin-web default - com.sun.jersey.spi.container.servlet.ServletContainer - - com.sun.jersey.config.property.packages - org.apache.zeppelin.rest - + org.glassfish.jersey.servlet.ServletContainer + + jersey.config.server.provider.packages + org.apache.zeppelin.rest + + 1 diff --git a/zeppelin-web/src/app/app.js b/zeppelin-web/src/app/app.js index aed7468631..b923aeb858 100644 --- a/zeppelin-web/src/app/app.js +++ b/zeppelin-web/src/app/app.js @@ -18,6 +18,12 @@ import 'headroom.js' import 'headroom.js/dist/angular.headroom' +import 'scrollmonitor/scrollMonitor.js' +import 'angular-viewport-watch/angular-viewport-watch.js' + +import 'angular-ui-grid/ui-grid.css' +import 'angular-ui-grid' + const requiredModules = [ 'ngCookies', 'ngAnimate', @@ -37,6 +43,17 @@ const requiredModules = [ 'focus-if', 'ngResource', 'ngclipboard', + 'angularViewportWatch', + 'ui.grid', + 'ui.grid.exporter', + 'ui.grid.edit', 'ui.grid.rowEdit', + 'ui.grid.selection', + 'ui.grid.cellNav', 'ui.grid.pinning', + 'ui.grid.grouping', + 'ui.grid.emptyBaseLayer', + 'ui.grid.resizeColumns', 'ui.grid.moveColumns', + 'ui.grid.pagination', + 'ui.grid.saveState', ] // headroom should not be used for CI, since we have to execute some integration tests. diff --git a/zeppelin-web/src/app/configuration/configuration.controller.js b/zeppelin-web/src/app/configuration/configuration.controller.js index e4f4e5d657..6b4f308fc5 100644 --- a/zeppelin-web/src/app/configuration/configuration.controller.js +++ b/zeppelin-web/src/app/configuration/configuration.controller.js @@ -34,7 +34,7 @@ function ConfigurationCtrl ($scope, $rootScope, $http, baseUrlSrv, ngToast) { timeout: '3000' }) setTimeout(function () { - window.location.replace('/') + window.location = baseUrlSrv.getBase() }, 3000) } console.log('Error %o %o', status, data.message) diff --git a/zeppelin-web/src/app/credential/credential.controller.js b/zeppelin-web/src/app/credential/credential.controller.js index eb649c8aa2..89cb1cfd36 100644 --- a/zeppelin-web/src/app/credential/credential.controller.js +++ b/zeppelin-web/src/app/credential/credential.controller.js @@ -40,7 +40,7 @@ function CredentialCtrl ($scope, $rootScope, $http, baseUrlSrv, ngToast) { timeout: '3000' }) setTimeout(function () { - window.location.replace('/') + window.location = baseUrlSrv.getBase() }, 3000) } console.log('Error %o %o', status, data.message) diff --git a/zeppelin-web/src/app/handsontable/handsonHelper.js b/zeppelin-web/src/app/handsontable/handsonHelper.js deleted file mode 100644 index 8d724c0e4b..0000000000 --- a/zeppelin-web/src/app/handsontable/handsonHelper.js +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * HandsonHelper class - */ -export default class HandsonHelper { - constructor (columns, rows, comment) { - this.columns = columns || [] - this.rows = rows || [] - this.comment = comment || '' - this._numericValidator = this._numericValidator.bind(this) - } - - getHandsonTableConfig (columns, columnNames, resultRows) { - let self = this - return { - colHeaders: columnNames, - data: resultRows, - rowHeaders: false, - stretchH: 'all', - sortIndicator: true, - columns: columns, - columnSorting: true, - contextMenu: false, - manualColumnResize: true, - manualRowResize: true, - readOnly: true, - readOnlyCellClassName: '', - fillHandle: false, - fragmentSelection: true, - disableVisualSelection: true, - cells: function (ro, co, pro) { - let cellProperties = {} - let colType = columns[co].type - cellProperties.renderer = function (instance, td, row, col, prop, value, cellProperties) { - self._cellRenderer(instance, td, row, col, prop, value, cellProperties, colType) - } - return cellProperties - }, - afterGetColHeader: function (col, TH) { - let instance = this - let menu = self._buildDropDownMenu(columns[col].type) - let button = self._buildTypeSwitchButton() - - self._addButtonMenuEvent(button, menu) - - Handsontable.Dom.addEvent(menu, 'click', function (event) { - if (event.target.nodeName === 'LI') { - self._setColumnType(columns, event.target.data.colType, instance, col) - } - }) - if (TH.firstChild.lastChild.nodeName === 'BUTTON') { - TH.firstChild.removeChild(TH.firstChild.lastChild) - } - TH.firstChild.appendChild(button) - TH.style['white-space'] = 'normal' - } - } - } - - /* - ** Private Service Functions - */ - - _addButtonMenuEvent (button, menu) { - Handsontable.Dom.addEvent(button, 'click', function (event) { - let changeTypeMenu - let position - let removeMenu - - document.body.appendChild(menu) - - event.preventDefault() - event.stopImmediatePropagation() - - changeTypeMenu = document.querySelectorAll('.changeTypeMenu') - - for (let i = 0, len = changeTypeMenu.length; i < len; i++) { - changeTypeMenu[i].style.display = 'none' - } - menu.style.display = 'block' - position = button.getBoundingClientRect() - - menu.style.top = (position.top + (window.scrollY || window.pageYOffset)) + 2 + 'px' - menu.style.left = (position.left) + 'px' - - removeMenu = function (event) { - if (menu.parentNode) { - menu.parentNode.removeChild(menu) - } - } - Handsontable.Dom.removeEvent(document, 'click', removeMenu) - Handsontable.Dom.addEvent(document, 'click', removeMenu) - }) - } - - _buildDropDownMenu (activeCellType) { - let menu = document.createElement('UL') - let types = ['text', 'numeric', 'date'] - let item - - menu.className = 'changeTypeMenu' - - for (let i = 0, len = types.length; i < len; i++) { - item = document.createElement('LI') - if ('innerText' in item) { - item.innerText = types[i] - } else { - item.textContent = types[i] - } - - item.data = {'colType': types[i]} - - if (activeCellType === types[i]) { - item.className = 'active' - } - menu.appendChild(item) - } - - return menu - } - - _buildTypeSwitchButton () { - let button = document.createElement('BUTTON') - - button.innerHTML = '\u25BC' - button.className = 'changeType' - - return button - } - - _isNumeric (value) { - if (!isNaN(value)) { - if (value.length !== 0) { - if (Number(value) <= Number.MAX_SAFE_INTEGER && Number(value) >= Number.MIN_SAFE_INTEGER) { - return true - } - } - } - return false - } - - _cellRenderer (instance, td, row, col, prop, value, cellProperties, colType) { - if (colType === 'numeric' && this._isNumeric(value)) { - cellProperties.format = '0,0.[00000]' - td.style.textAlign = 'left' - // eslint-disable-next-line prefer-rest-params - Handsontable.renderers.NumericRenderer.apply(this, arguments) - } else if (value.length > '%html'.length && value.substring(0, '%html '.length) === '%html ') { - td.innerHTML = value.substring('%html'.length) - } else { - // eslint-disable-next-line prefer-rest-params - Handsontable.renderers.TextRenderer.apply(this, arguments) - } - } - - _dateValidator (value, callback) { - let d = moment(value) - return callback(d.isValid()) - } - - _numericValidator (value, callback) { - return callback(this._isNumeric(value)) - } - - _setColumnType (columns, type, instance, col) { - columns[col].type = type - this._setColumnValidator(columns, col) - instance.updateSettings({columns: columns}) - instance.validateCells(null) - if (this._isColumnSorted(instance, col)) { - instance.sort(col, instance.sortOrder) - } - } - - _isColumnSorted (instance, col) { - return instance.sortingEnabled && instance.sortColumn === col - } - - _setColumnValidator (columns, col) { - if (columns[col].type === 'numeric') { - columns[col].validator = this._numericValidator - } else if (columns[col].type === 'date') { - columns[col].validator = this._dateValidator - } else { - columns[col].validator = null - } - } -} diff --git a/zeppelin-web/src/app/home/home.css b/zeppelin-web/src/app/home/home.css index ce30e6642d..03ae4c1576 100644 --- a/zeppelin-web/src/app/home/home.css +++ b/zeppelin-web/src/app/home/home.css @@ -32,14 +32,14 @@ body { background: white; } -.displayNavBar { - display: inline !important; -} - body.asIframe { padding-top: 0; } +.displayNavBar { + display: inline !important; +} + body .navbar { margin-bottom: 0; } @@ -59,58 +59,6 @@ body .navbar { text-decoration: none; } -.navbar-inverse { - background: #3071a9; - color: #fff; - border-color: #3071a9; - font-size: 18px; -} - -.navbar-inverse .navbar-nav > .open > a, -.navbar-inverse .navbar-nav > .open > a:hover, -.navbar-inverse .navbar-nav > .open > a:focus { - color: #fff; - background: #3071a9; -} - -.navbar-inverse .navbar-toggle:hover, -.navbar-inverse .navbar-toggle:focus { - background: #3071a9; -} - -.navbar-inverse .navbar-nav > li > a:hover, -.navbar-inverse .navbar-nav > li > a:focus { - color: #fff; - background: rgba(0, 0, 0, 0.2); - -} - -.navbar-inverse .navbar-toggle { - border-color: #FFFFFF; -} - -.navbar-inverse .navbar-nav > .active > a, -.navbar-inverse .navbar-nav > .active > a:hover, -.navbar-inverse .navbar-nav > .active > a:focus { - color: #fff; - background: #080808; -} - -.navbar-inverse .navbar-nav > li > a { - color: #FFFFFF; -} - -.navbar-inverse .navbar-brand { - color: #fff; - text-decoration: none; - font-family: 'Patua One', cursive; - font-size: 32px; -} - -a.navbar-brand:hover { - color: #fff !important; -} - /* Css for the Notebook Dropdown */ .expandable ul { @@ -248,45 +196,6 @@ a.navbar-brand:hover { border-radius:6px 0 6px 6px; } -@media (max-width: 767px) { - .navbar-inverse .navbar-nav .open .dropdown-menu > li > a, - #notebook-list li a { - color: #D3D3D3; - } - - .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a, - #notebook-list li a { - padding: 5px 15px 5px 25px; - line-height: 20px; - } - - .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a { - color: #D3D3D3; - } - - .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a:hover, - .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a:focus, - #notebook-list li a:hover { - color: #fff; - background: transparent; - } - - .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a, - .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a:hover, - .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a:focus { - color: #fff; - background: #080808; - } - - .navbar-inverse .navbar-nav .open .dropdown-menu .divider { - background: #3071A9; - } - - .navbar-inverse .navbar-collapse, .navbar-inverse .navbar-form { - border-color: #3071A9; - } -} - #main { padding: 10px; height: 100%; @@ -335,14 +244,6 @@ a.navbar-brand:hover { border-color: #357ebd; } -.username { - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - max-width: 120px; - display: inline-block; -} - .server-connected { padding-top: 12px; color: #00CC00; @@ -455,40 +356,6 @@ This part should be removed when new version of bootstrap handles this issue. z-index: 10003 !important; } -#noteImportModal .modal-body { - min-height: 420px; - overflow: hidden; -} - -#noteImportModal .modal-footer { - min-height: 65px; -} - -#noteImportModal .display-inline a { - background: #fff; - border: 1px solid #ddd; - border-radius: 5px; - color: #999; - cursor: pointer; - display: block; - float: left; - font-size: 98px; - height: 240px; - margin: 0 10px 16px; - padding-top: 60px; - text-align: center; - text-decoration: none; - width: 264px; -} - -#noteImportModal .display-inline a:hover { - background: #eee; -} - -#noteImportModal .display-inline a p { - font-size: 14px; -} - /* ------------------------------------------- */ /* Slide Top /* ------------------------------------------- */ @@ -824,28 +691,3 @@ This part should be removed when new version of bootstrap handles this issue. .bootstrap-dialog.type-primary .modal-header { background: #3071a9; } - -/* About Zeppelin */ -.about { - height: 200px; - padding: 25px; -} - -.about .logo { - padding-top: 30px; -} - -.about .logo img { - width: 95%; -} - -.about .content { - text-align: center; -} - -.about .content h3 { - font-family: 'Patua One'; - color: #3071A9; - font-size: 30px; - margin: 0 auto; -} diff --git a/zeppelin-web/src/app/home/notebook-template.html b/zeppelin-web/src/app/home/notebook-template.html index 0456a39ec6..4c287de69c 100644 --- a/zeppelin-web/src/app/home/notebook-template.html +++ b/zeppelin-web/src/app/home/notebook-template.html @@ -56,7 +56,7 @@ limitations under the License.

    - {{getNoteName(node)}} + {{getNoteName(node)}} diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 330c2adadf..6afc077c13 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -101,7 +101,7 @@ function InterpreterCtrl ($rootScope, $scope, $http, baseUrlSrv, ngToast, $timeo timeout: '3000' }) setTimeout(function () { - window.location.replace('/') + window.location = baseUrlSrv.getBase() }, 3000) } console.log('Error %o %o', status, data.message) diff --git a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js index e1ce02ddc2..e811f7bdbd 100644 --- a/zeppelin-web/src/app/jobmanager/jobs/job.controller.js +++ b/zeppelin-web/src/app/jobmanager/jobs/job.controller.js @@ -12,6 +12,8 @@ * limitations under the License. */ +import moment from 'moment' + import { ParagraphStatus, } from '../../notebook/paragraph/paragraph.status' angular.module('zeppelinWebApp').controller('JobCtrl', JobCtrl) diff --git a/zeppelin-web/src/app/notebook/notebook-actionBar.html b/zeppelin-web/src/app/notebook/notebook-actionBar.html index e83e05e190..5b600f2611 100644 --- a/zeppelin-web/src/app/notebook/notebook-actionBar.html +++ b/zeppelin-web/src/app/notebook/notebook-actionBar.html @@ -156,7 +156,7 @@ limitations under the License. {{revision.message}} - {{moment.unix(revision.time).format('MMMM Do YYYY, h:mm a')}} + {{formatRevisionDate(revision.time)}} diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index 4b02f1a979..a5126642d0 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -12,6 +12,8 @@ * limitations under the License. */ +import moment from 'moment' + import { isParagraphRunning, } from './paragraph/paragraph.status' angular.module('zeppelinWebApp').controller('NotebookCtrl', NotebookCtrl) @@ -25,7 +27,6 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, ngToast.dismiss() $scope.note = null - $scope.moment = moment $scope.editorToggled = false $scope.tableToggled = false $scope.viewOnly = false @@ -42,6 +43,10 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, {name: '1d', value: '0 0 0 * * ?'} ] + $scope.formatRevisionDate = function(date) { + return moment.unix(date).format('MMMM Do YYYY, h:mm a') + } + $scope.interpreterSettings = [] $scope.interpreterBindings = [] $scope.isNoteDirty = null @@ -223,7 +228,7 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, } $scope.$on('listRevisionHistory', function (event, data) { - console.log('received list of revisions %o', data) + console.debug('received list of revisions %o', data) $scope.noteRevisions = data.revisionList $scope.noteRevisions.splice(0, 0, { id: 'Head', @@ -464,10 +469,12 @@ function NotebookCtrl ($scope, $route, $routeParams, $location, $rootScope, let addPara = function (paragraph, index) { $scope.note.paragraphs.splice(index, 0, paragraph) - _.each($scope.note.paragraphs, function (para) { + $scope.note.paragraphs.map(para => { if (para.id === paragraph.id) { para.focus = true - $scope.$broadcast('focusParagraph', para.id, 0, false) + + // we need `$timeout` since angular DOM might not be initialized + $timeout(() => { $scope.$broadcast('focusParagraph', para.id, 0, false) }) } }) } diff --git a/zeppelin-web/src/app/notebook/notebook.html b/zeppelin-web/src/app/notebook/notebook.html index 293e2c2bd6..63ef98b94d 100644 --- a/zeppelin-web/src/app/notebook/notebook.html +++ b/zeppelin-web/src/app/notebook/notebook.html @@ -112,7 +112,8 @@ limitations under the License. ng-controller="ParagraphCtrl" ng-init="init(currentParagraph, note)" ng-class="columnWidthClass(currentParagraph.config.colWidth)" - style="margin: 0; padding: 0;"> + style="margin: 0; padding: 0;" + viewport-watch>

    +

    diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-control.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-control.html index 222b3b3a6a..5523609eb9 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-control.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-control.html @@ -44,11 +44,12 @@ limitations under the License. - - diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index a6564d4a0d..00be23efbb 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -17,6 +17,9 @@ import { ParagraphStatus, isParagraphRunning, } from './paragraph.status' +import moment from 'moment' +require('moment-duration-format') + const ParagraphExecutor = { SPELL: 'SPELL', INTERPRETER: 'INTERPRETER', @@ -251,12 +254,15 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca $scope.propagateSpellResult = function (paragraphId, paragraphTitle, paragraphText, paragraphResults, paragraphStatus, paragraphErrorMessage, - paragraphConfig, paragraphSettingsParam) { + paragraphConfig, paragraphSettingsParam, + paragraphDateStarted, paragraphDateFinished) { websocketMsgSrv.paragraphExecutedBySpell( paragraphId, paragraphTitle, paragraphText, paragraphResults, paragraphStatus, paragraphErrorMessage, - paragraphConfig, paragraphSettingsParam) + paragraphConfig, paragraphSettingsParam, + paragraphDateStarted, paragraphDateFinished + ) } $scope.handleSpellError = function (paragraphText, error, @@ -266,11 +272,16 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca $scope.paragraph.errorMessage = errorMessage console.error('Failed to execute interpret() in spell\n', error) + if (!propagated) { + $scope.paragraph.dateFinished = $scope.getFormattedParagraphTime() + } + if (!propagated) { $scope.propagateSpellResult( $scope.paragraph.id, $scope.paragraph.title, paragraphText, [], $scope.paragraph.status, errorMessage, - $scope.paragraph.config, $scope.paragraph.settings.params) + $scope.paragraph.config, $scope.paragraph.settings.params, + $scope.paragraph.dateStarted, $scope.paragraph.dateFinished) } } @@ -305,12 +316,17 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca const resultsMsg = $scope.spellTransaction.resultsMsg const paragraphText = $scope.spellTransaction.paragraphText + if (!propagated) { + $scope.paragraph.dateFinished = $scope.getFormattedParagraphTime() + } + if (!propagated) { const propagable = SpellResult.createPropagable(resultsMsg) $scope.propagateSpellResult( $scope.paragraph.id, $scope.paragraph.title, paragraphText, propagable, status, '', - $scope.paragraph.config, $scope.paragraph.settings.params) + $scope.paragraph.config, $scope.paragraph.settings.params, + $scope.paragraph.dateStarted, $scope.paragraph.dateFinished) } } @@ -328,6 +344,10 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca // remove leading spaces const textWithoutMagic = splited[1].replace(/^\s+/g, '') + if (!propagated) { + $scope.paragraph.dateStarted = $scope.getFormattedParagraphTime() + } + // handle actual result message in promise heliumService.executeSpell(magic, textWithoutMagic) .then(resultsMsg => { @@ -627,10 +647,10 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca $scope.editor.renderer.setShowGutter($scope.paragraph.config.lineNumbers) $scope.editor.setShowFoldWidgets(false) $scope.editor.setHighlightActiveLine(false) - $scope.editor.setHighlightGutterLine(false) $scope.editor.getSession().setUseWrapMode(true) $scope.editor.setTheme('ace/theme/chrome') $scope.editor.setReadOnly($scope.isRunning($scope.paragraph)) + $scope.editor.setHighlightActiveLine($scope.paragraphFocused) if ($scope.paragraphFocused) { let prefix = '%' + getInterpreterName($scope.paragraph.text) @@ -815,20 +835,16 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca switch (keyCode) { case 38: - keyBindingEditorFocusAction(ROW_UP) + if (!e.shiftKey) { keyBindingEditorFocusAction(ROW_UP) } break case 80: - if (e.ctrlKey && !e.altKey) { - keyBindingEditorFocusAction(ROW_UP) - } + if (e.ctrlKey && !e.altKey) { keyBindingEditorFocusAction(ROW_UP) } break case 40: - keyBindingEditorFocusAction(ROW_DOWN) + if (!e.shiftKey) { keyBindingEditorFocusAction(ROW_DOWN) } break case 78: - if (e.ctrlKey && !e.altKey) { - keyBindingEditorFocusAction(ROW_DOWN) - } + if (e.ctrlKey && !e.altKey) { keyBindingEditorFocusAction(ROW_DOWN) } break } } @@ -837,8 +853,11 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca } } - const handleFocus = function (value, isDigestPass) { - $scope.paragraphFocused = value + const handleFocus = function (focused, isDigestPass) { + $scope.paragraphFocused = focused + + if ($scope.editor) { $scope.editor.setHighlightActiveLine(focused) } + if (isDigestPass === false || isDigestPass === undefined) { // Protect against error in case digest is already running $timeout(function () { @@ -982,20 +1001,29 @@ function ParagraphCtrl ($scope, $rootScope, $route, $window, $routeParams, $loca return $scope.currentProgress || 0 } + $scope.getFormattedParagraphTime = () => { + return moment().toISOString() + } + $scope.getExecutionTime = function (pdata) { - let timeMs = Date.parse(pdata.dateFinished) - Date.parse(pdata.dateStarted) + const end = pdata.dateFinished + const start = pdata.dateStarted + let timeMs = Date.parse(end) - Date.parse(start) if (isNaN(timeMs) || timeMs < 0) { if ($scope.isResultOutdated(pdata)) { return 'outdated' } return '' } + + const durationFormat = moment.duration((timeMs / 1000), 'seconds').format('h [hrs] m [min] s [sec]') + const endFormat = moment(pdata.dateFinished).format('MMMM DD YYYY, h:mm:ss A') + let user = (pdata.user === undefined || pdata.user === null) ? 'anonymous' : pdata.user - let desc = 'Took ' + moment.duration((timeMs / 1000), 'seconds').format('h [hrs] m [min] s [sec]') + - '. Last updated by ' + user + ' at ' + moment(pdata.dateFinished).format('MMMM DD YYYY, h:mm:ss A') + '.' - if ($scope.isResultOutdated(pdata)) { - desc += ' (outdated)' - } + let desc = `Took ${durationFormat}. Last updated by ${user} at ${endFormat}.` + + if ($scope.isResultOutdated(pdata)) { desc += ' (outdated)' } + return desc } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.css b/zeppelin-web/src/app/notebook/paragraph/paragraph.css index e517be42fe..8ecc7bba75 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.css +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.css @@ -52,21 +52,6 @@ width: 100%; } -table.dataTable { - margin-top: 0px !important; - margin-bottom: 6px !important; -} - -table.dataTable.table-condensed > thead > tr > th { - padding-right: 28px; -} - -table.dataTable.table-condensed .sorting:after, -table.dataTable.table-condensed .sorting_asc:after, -table.dataTable.table-condensed .sorting_desc:after { - right: 12px; -} - .plainTextContainer { font-family: "Monaco","Menlo","Ubuntu Mono","Consolas","source-code-pro",monospace; font-size: 12px !important; @@ -128,7 +113,6 @@ table.dataTable.table-condensed .sorting_desc:after { display: none; float: right; color: #999; - margin-top: -9px; margin-right: 0; position: absolute; clear: both; @@ -190,8 +174,9 @@ table.dataTable.table-condensed .sorting_desc:after { } .paragraph .control { - background: rgba(255,255,255,0.85); + position: absolute; float: right; + background: rgba(255,255,255,0.85); color: #999; margin-top: 1px; margin-right: 5px; @@ -385,10 +370,22 @@ table.table-shortcut { direction: ltr; } +/** set highlight line color */ +#main .ace-chrome .ace_marker-layer .ace_active-line { + background: rgba(114, 127, 222, 0.08); +} + +/** hide not focused cursors */ .ace_hidden-cursors { opacity: 0; } +/** set cursor color */ +#main .emacs-mode .ace_cursor { + background: #C0C0C0!important; + border: none !important; +} + .ace_text-input, .ace_gutter, .ace_layer, .emacs-mode, .ace_text-layer, .ace_cursor-layer, .ace_cursor, .ace_scrollbar { @@ -399,11 +396,6 @@ table.table-shortcut { opacity: 0 !important; } -#main .emacs-mode .ace_cursor { - background: #C0C0C0!important; - border: none !important; -} - /* Table Display CSS */ @@ -487,46 +479,6 @@ table.table-shortcut { background: none; } -/* - Handsontable -*/ - -.handsontable th { - font-weight: bold; -} - -.handsontable th, .handsontable td { - border-right: 0px; - border-left: 0px !important; - padding: 4px; -} - -.handsontable tr:first-child th { - text-align: left; - border-top: 0px; - padding: 4px 0px 0px 0px; - border-left: 0px; - border-right: 0px; - border-bottom: 2px solid #CCC; -} - -.handsontable .columnSorting.ascending::after { - content: '\f160'; - margin-left: 3px; - font-size: 12px; - font-family: FontAwesome; - line-height: 2; -} - -.handsontable .columnSorting.descending::after { - content: '\f161'; - margin-left: 3px; - font-size: 12px; - margin-left: 3px; - font-family: FontAwesome; - line-height: 2; -} - /* Pivot CSS */ @@ -583,54 +535,6 @@ table.table-striped { width: 20px; } - -.changeType { - border: 1px solid #bbb; - color: #bbb; - background: #eee; - border-radius: 2px; - padding: 2px; - font-size: 9px; - float: right; - line-height: 9px; - margin: 3px 3px 0 0; -} -.changeType:hover { - border: 1px solid #777; - color: #777; - cursor: pointer; -} -.changeType.pressed { - background-color: #999; -} -.changeTypeMenu { - position: absolute; - border: 1px solid #ccc; - margin-top: 22px; - box-shadow: 0 1px 3px -1px #323232; - background: white; - padding: 0; - font-size: 13px; - display: none; - z-index: 10; -} -.changeTypeMenu li { - text-align: left; - list-style: none; - padding: 2px 20px; - cursor: pointer; - margin-bottom: 0; -} -.changeTypeMenu li.active:before { - font-size: 12px; - content: "\2714"; - margin-left: -15px; - margin-right: 3px; -} -.changeTypeMenu li:hover { - background: #eee; -} - /* Overwrite github-markdown-css for Markdown interpreter */ @@ -660,3 +564,7 @@ table.table-striped { .markdown-body h4 { font-size: 16px; } + +.network-labels { + margin: 0.2em; +} diff --git a/zeppelin-web/src/app/notebook/paragraph/result/display-table.css b/zeppelin-web/src/app/notebook/paragraph/result/display-table.css new file mode 100644 index 0000000000..946210913b --- /dev/null +++ b/zeppelin-web/src/app/notebook/paragraph/result/display-table.css @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.ui-grid-pager-panel { + vertical-align: middle; + background-color: #f3f3f3; +} + +.ui-grid-footer-info { + padding: 5px; + background-color: #f3f3f3; +} + +.ui-grid-menu { + overflow: auto; /** enable scrollbar in column menu */ +} + +/** disable alternative color for even lows in a table */ +.ui-grid-row:nth-child(even) .ui-grid-cell { + background-color: white !important; +} + +/** support `height: auto` for cells */ +.ui-grid-viewport .ui-grid-cell-contents { + word-wrap: break-word; + white-space: normal !important; + border-bottom: 1px solid #e8e8e8; +} + +.ui-grid-row, .ui-grid-cell { + height: auto !important; +} + +.ui-grid-row div[role=row] { + display: flex ; + align-content: stretch; +} + diff --git a/zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html b/zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html index d8a316b4b2..9592d809d3 100644 --- a/zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html +++ b/zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html @@ -13,12 +13,13 @@ limitations under the License. -->
    -
    +
    + +
    + +
    + +
    + +
    \ No newline at end of file diff --git a/zeppelin-web/src/app/tabledata/networkdata.js b/zeppelin-web/src/app/tabledata/networkdata.js new file mode 100644 index 0000000000..7983d82726 --- /dev/null +++ b/zeppelin-web/src/app/tabledata/networkdata.js @@ -0,0 +1,145 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import TableData from './tabledata' +import {DatasetType} from './dataset' + +/** + * Create network data object from paragraph graph type result + */ +export default class NetworkData extends TableData { + constructor(graph) { + super() + this.graph = graph || {} + if (this.graph.nodes) { + this.loadParagraphResult({msg: JSON.stringify(graph), type: DatasetType.NETWORK}) + } + } + + loadParagraphResult(paragraphResult) { + if (!paragraphResult || paragraphResult.type !== DatasetType.NETWORK) { + console.log('Can not load paragraph result') + return + } + + this.graph = JSON.parse(paragraphResult.msg.trim() || '{}') + + if (!this.graph.nodes) { + console.log('Graph result is empty') + return + } + + this.setNodesDefaults() + this.setEdgesDefaults() + + this.networkNodes = angular.equals({}, this.graph.labels || {}) + ? null : {count: this.graph.nodes.length, labels: this.graph.labels} + this.networkRelationships = angular.equals([], this.graph.types || []) + ? null : {count: this.graph.edges.length, types: this.graph.types} + + let rows = [] + let comment = '' + let entities = this.graph.nodes.concat(this.graph.edges) + let baseColumnNames = [{name: 'id', index: 0, aggr: 'sum'}, + {name: 'label', index: 1, aggr: 'sum'}] + let internalFieldsToJump = ['count', 'size', 'totalCount', + 'data', 'x', 'y', 'labels'] + let baseCols = _.map(baseColumnNames, function(col) { return col.name }) + let keys = _.map(entities, function(elem) { return Object.keys(elem.data || {}) }) + keys = _.flatten(keys) + keys = _.uniq(keys).filter(function(key) { + return baseCols.indexOf(key) === -1 + }) + let columnNames = baseColumnNames.concat(_.map(keys, function(elem, i) { + return {name: elem, index: i + baseColumnNames.length, aggr: 'sum'} + })) + for (let i = 0; i < entities.length; i++) { + let entity = entities[i] + let col = [] + let col2 = [] + entity.data = entity.data || {} + for (let j = 0; j < columnNames.length; j++) { + let name = columnNames[j].name + let value = name in entity && internalFieldsToJump.indexOf(name) === -1 + ? entity[name] : entity.data[name] + let parsedValue = value === null || value === undefined ? '' : value + col.push(parsedValue) + col2.push({key: name, value: parsedValue}) + } + rows.push(col) + } + + this.comment = comment + this.columns = columnNames + this.rows = rows + } + + setNodesDefaults() { + } + + setEdgesDefaults() { + this.graph.edges + .sort((a, b) => { + if (a.source > b.source) { + return 1 + } else if (a.source < b.source) { + return -1 + } else if (a.target > b.target) { + return 1 + } else if (a.target < b.target) { + return -1 + } else { + return 0 + } + }) + this.graph.edges + .forEach((edge, index) => { + let prevEdge = this.graph.edges[index - 1] + edge.count = (index > 0 && +edge.source === +prevEdge.source && +edge.target === +prevEdge.target + ? prevEdge.count : 0) + 1 + edge.totalCount = this.graph.edges + .filter((innerEdge) => +edge.source === +innerEdge.source && +edge.target === +innerEdge.target) + .length + }) + this.graph.edges + .forEach((edge) => { + if (typeof +edge.source === 'number') { + edge.source = this.graph.nodes.filter((node) => +edge.source === +node.id)[0] || null + } + if (typeof +edge.target === 'number') { + edge.target = this.graph.nodes.filter((node) => +edge.target === +node.id)[0] || null + } + }) + } + + getNetworkProperties() { + let baseCols = ['id', 'label'] + let properties = {} + this.graph.nodes.forEach(function(node) { + let hasLabel = 'label' in node && node.label !== '' + if (!hasLabel) { + return + } + let label = node.label + let hasKey = hasLabel && label in properties + let keys = _.uniq(Object.keys(node.data || {}) + .concat(hasKey ? properties[label].keys : baseCols)) + if (!hasKey) { + properties[label] = {selected: 'label'} + } + properties[label].keys = keys + }) + return properties + } +} diff --git a/zeppelin-web/src/app/tabledata/networkdata.test.js b/zeppelin-web/src/app/tabledata/networkdata.test.js new file mode 100644 index 0000000000..f8d98a89a3 --- /dev/null +++ b/zeppelin-web/src/app/tabledata/networkdata.test.js @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import NetworkData from './networkdata.js' +import {DatasetType} from './dataset.js' + +describe('NetworkData build', function() { + let nd + + beforeEach(function() { + nd = new NetworkData() + }) + + it('should initialize the default value', function() { + expect(nd.columns.length).toBe(0) + expect(nd.rows.length).toBe(0) + expect(nd.graph).toEqual({}) + }) + + it('should able to create NetowkData from paragraph result', function() { + let jsonExpected = {nodes: [{id: 1}, {id: 2}], edges: [{source: 2, target: 1, id: 1}]} + nd.loadParagraphResult({ + type: DatasetType.NETWORK, + msg: JSON.stringify(jsonExpected) + }) + + expect(nd.columns.length).toBe(2) + expect(nd.rows.length).toBe(3) + expect(nd.graph.nodes[0].id).toBe(jsonExpected.nodes[0].id) + expect(nd.graph.nodes[1].id).toBe(jsonExpected.nodes[1].id) + expect(nd.graph.edges[0].id).toBe(jsonExpected.edges[0].id) + expect(nd.graph.edges[0].source.id).toBe(jsonExpected.nodes[1].id) + expect(nd.graph.edges[0].target.id).toBe(jsonExpected.nodes[0].id) + }) +}) diff --git a/zeppelin-web/src/app/tabledata/tabledata.js b/zeppelin-web/src/app/tabledata/tabledata.js index 8e4e6b6625..3fe01b7791 100644 --- a/zeppelin-web/src/app/tabledata/tabledata.js +++ b/zeppelin-web/src/app/tabledata/tabledata.js @@ -11,19 +11,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import {Dataset, DatasetType} from './dataset' /** * Create table data object from paragraph table type result */ -export default class TableData { +export default class TableData extends Dataset { constructor (columns, rows, comment) { + super() this.columns = columns || [] this.rows = rows || [] this.comment = comment || '' } loadParagraphResult (paragraphResult) { - if (!paragraphResult || paragraphResult.type !== 'TABLE') { + if (!paragraphResult || paragraphResult.type !== DatasetType.TABLE) { console.log('Can not load paragraph result') return } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-areachart.js b/zeppelin-web/src/app/visualization/builtins/visualization-areachart.js index 900f0f41da..494f8ae67f 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-areachart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-areachart.js @@ -23,6 +23,12 @@ export default class AreachartVisualization extends Nvd3ChartVisualization { super(targetEl, config) this.pivot = new PivotTransformation(config) + + try { + this.config.rotate = {degree: config.rotate.degree} + } catch (e) { + this.config.rotate = {degree: '-45'} + } } type () { @@ -46,6 +52,7 @@ export default class AreachartVisualization extends Nvd3ChartVisualization { this.xLabels = d3Data.xLabels super.render(d3Data) + this.config.changeXLabel(this.config.xLabelStatus) } /** @@ -58,11 +65,61 @@ export default class AreachartVisualization extends Nvd3ChartVisualization { configureChart (chart) { let self = this + let configObj = self.config + chart.xAxis.tickFormat(function (d) { return self.xAxisTickFormat(d, self.xLabels) }) chart.yAxis.tickFormat(function (d) { return self.yAxisTickFormat(d) }) chart.yAxis.axisLabelDistance(50) chart.useInteractiveGuideline(true) // for better UX and performance issue. (https://github.com/novus/nvd3/issues/691) + self.config.changeXLabel = function(type) { + switch (type) { + case 'default': + self.chart._options['showXAxis'] = true + self.chart._options['margin'] = {bottom: 50} + self.chart.xAxis.rotateLabels(0) + configObj.xLabelStatus = 'default' + break + case 'rotate': + self.chart._options['showXAxis'] = true + self.chart._options['margin'] = {bottom: 140} + self.chart.xAxis.rotateLabels(configObj.rotate.degree) + configObj.xLabelStatus = 'rotate' + break + case 'hide': + self.chart._options['showXAxis'] = false + self.chart._options['margin'] = {bottom: 50} + d3.select('#' + self.targetEl[0].id + '> svg').select('g.nv-axis.nv-x').selectAll('*').remove() + configObj.xLabelStatus = 'hide' + break + } + self.emitConfig(configObj) + } + + self.config.isXLabelStatus = function(type) { + if (configObj.xLabelStatus === type) { + return true + } else { + return false + } + } + + self.config.setDegree = function(type) { + configObj.rotate.degree = type + self.chart.xAxis.rotateLabels(type) + self.emitConfig(configObj) + } + + self.config.isDegreeEmpty = function() { + if (configObj.rotate.degree.length > 0) { + return true + } else { + configObj.rotate.degree = '-45' + self.emitConfig(configObj) + return false + } + } + this.chart.style(this.config.style || 'stack') this.chart.dispatch.on('stateChange', function (s) { @@ -74,4 +131,25 @@ export default class AreachartVisualization extends Nvd3ChartVisualization { }, 500) }) } + getSetting(chart) { + let self = this + let configObj = self.config + + // default to visualize xLabel + if (typeof (configObj.xLabelStatus) === 'undefined') { + configObj.changeXLabel('default') + } + + if (typeof (configObj.rotate.degree) === 'undefined' || configObj.rotate.degree === '') { + configObj.rotate.degree = '-45' + self.emitConfig(configObj) + } + + return { + template: 'app/visualization/builtins/visualization-displayXAxis.html', + scope: { + config: configObj + } + } + } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-barchart.js b/zeppelin-web/src/app/visualization/builtins/visualization-barchart.js index 258fcce55b..2653af21e7 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-barchart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-barchart.js @@ -23,6 +23,12 @@ export default class BarchartVisualization extends Nvd3ChartVisualization { super(targetEl, config) this.pivot = new PivotTransformation(config) + + try { + this.config.rotate = {degree: config.rotate.degree} + } catch (e) { + this.config.rotate = {degree: '-45'} + } } type () { @@ -76,7 +82,7 @@ export default class BarchartVisualization extends Nvd3ChartVisualization { case 'rotate': self.chart._options['showXAxis'] = true self.chart._options['margin'] = {bottom: 140} - self.chart.xAxis.rotateLabels(-45) + self.chart.xAxis.rotateLabels(configObj.rotate.degree) configObj.xLabelStatus = 'rotate' break case 'hide': @@ -86,6 +92,7 @@ export default class BarchartVisualization extends Nvd3ChartVisualization { configObj.xLabelStatus = 'hide' break } + self.emitConfig(configObj) } self.config.isXLabelStatus = function(type) { @@ -96,6 +103,12 @@ export default class BarchartVisualization extends Nvd3ChartVisualization { } } + self.config.setDegree = function(type) { + configObj.rotate.degree = type + self.chart.xAxis.rotateLabels(type) + self.emitConfig(configObj) + } + this.chart.dispatch.on('stateChange', function(s) { configObj.stacked = s.stacked @@ -115,39 +128,15 @@ export default class BarchartVisualization extends Nvd3ChartVisualization { configObj.changeXLabel('default') } + if (typeof (configObj.rotate.degree) === 'undefined' || configObj.rotate.degree === '') { + configObj.rotate.degree = '-45' + self.emitConfig(configObj) + } + return { - template: `
    - xAxis : -
    - -
    - - - - - -
    `, + template: 'app/visualization/builtins/visualization-displayXAxis.html', scope: { - config: configObj, - save: function(type) { - configObj.changeXLabel(type) - self.emitConfig(configObj) - } + config: configObj } } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js b/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js new file mode 100644 index 0000000000..506b1c5f18 --- /dev/null +++ b/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js @@ -0,0 +1,263 @@ +/* + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Visualization from '../visualization' +import NetworkTransformation from '../../tabledata/network' + +/** + * Visualize data in network format + */ +export default class NetworkVisualization extends Visualization { + constructor(targetEl, config) { + super(targetEl, config) + console.log('Init network viz') + if (!config.properties) { + config.properties = {} + } + if (!config.d3Graph) { + config.d3Graph = { + forceLayout: { + timeout: 10000, + charge: -120, + linkDistance: 80, + }, + zoom: { + minScale: 1.3 + } + } + } + this.targetEl.addClass('network') + this.containerId = this.targetEl.prop('id') + this.force = null + this.svg = null + this.$timeout = angular.injector(['ng']).get('$timeout') + this.$interpolate = angular.injector(['ng']).get('$interpolate') + this.transformation = new NetworkTransformation(config) + } + + refresh() { + console.log('refresh') + } + + render(networkData) { + if (!('graph' in networkData)) { + console.log('graph not found') + return + } + console.log('Render Graph Visualization') + + let transformationConfig = this.transformation.getSetting().scope.config + console.log('cfg', transformationConfig) + if (transformationConfig && angular.equals({}, transformationConfig.properties)) { + transformationConfig.properties = networkData.getNetworkProperties() + } + + this.targetEl.empty().append('') + + let width = this.targetEl.width() + let height = this.targetEl.height() + let self = this + let defaultOpacity = 0 + let nodeSize = 10 + let textOffset = 3 + let linkSize = 10 + + let arcPath = (leftHand, d) => { + let start = leftHand ? d.source : d.target + let end = leftHand ? d.target : d.source + let dx = end.x - start.x + let dy = end.y - start.y + let dr = d.totalCount === 1 + ? 0 : Math.sqrt(Math.pow(dx, 2) + Math.pow(dy, 2)) / (1 + (1 / d.totalCount) * (d.count - 1)) + let sweep = leftHand ? 0 : 1 + return `M${start.x},${start.y}A${dr},${dr} 0 0,${sweep} ${end.x},${end.y}` + } + // Use elliptical arc path segments to doubly-encode directionality. + let tick = () => { + // Links + linkPath.attr('d', function(d) { + return arcPath(true, d) + }) + textPath.attr('d', function(d) { + return arcPath(d.source.x < d.target.x, d) + }) + // Nodes + circle.attr('transform', (d) => `translate(${d.x},${d.y})`) + text.attr('transform', (d) => `translate(${d.x},${d.y})`) + } + + let setOpacity = (scale) => { + let opacity = scale >= +transformationConfig.d3Graph.zoom.minScale ? 1 : 0 + this.svg.selectAll('.nodeLabel') + .style('opacity', opacity) + this.svg.selectAll('textPath') + .style('opacity', opacity) + } + + let zoom = d3.behavior.zoom() + .scaleExtent([1, 10]) + .on('zoom', () => { + console.log('zoom') + setOpacity(d3.event.scale) + container.attr('transform', `translate(${d3.event.translate})scale(${d3.event.scale})`) + }) + + this.svg = d3.select(`#${this.containerId} svg`) + .attr('width', width) + .attr('height', height) + .call(zoom) + + this.force = d3.layout.force() + .charge(transformationConfig.d3Graph.forceLayout.charge) + .linkDistance(transformationConfig.d3Graph.forceLayout.linkDistance) + .on('tick', tick) + .nodes(networkData.graph.nodes) + .links(networkData.graph.edges) + .size([width, height]) + .on('start', () => { + console.log('force layout start') + this.$timeout(() => { this.force.stop() }, transformationConfig.d3Graph.forceLayout.timeout) + }) + .on('end', () => { + console.log('force layout stop') + setOpacity(zoom.scale()) + }) + .start() + + let renderFooterOnClick = (entity, type) => { + let footerId = this.containerId + '_footer' + let obj = {id: entity.id, label: entity.defaultLabel || entity.label, type: type} + let html = [this.$interpolate(['
  • {{type}}_id: {{id}}
  • ', + '
  • {{type}}_type: {{label}}
  • '].join(''))(obj)] + html = html.concat(_.map(entity.data, (v, k) => { + return this.$interpolate('
  • {{field}}: {{value}}
  • ')({field: k, value: v}) + })) + angular.element('#' + footerId) + .find('.list-inline') + .empty() + .append(html.join('')) + } + + let drag = d3.behavior.drag() + .origin((d) => d) + .on('dragstart', function(d) { + console.log('dragstart') + d3.event.sourceEvent.stopPropagation() + d3.select(this).classed('dragging', true) + self.force.stop() + }) + .on('drag', function(d) { + console.log('drag') + d.px += d3.event.dx + d.py += d3.event.dy + d.x += d3.event.dx + d.y += d3.event.dy + }) + .on('dragend', function(d) { + console.log('dragend') + d.fixed = true + d3.select(this).classed('dragging', false) + self.force.resume() + }) + + let container = this.svg.append('g') + if (networkData.graph.directed) { + container.append('svg:defs').selectAll('marker') + .data(['arrowMarker-' + this.containerId]) + .enter() + .append('svg:marker') + .attr('id', String) + .attr('viewBox', '0 -5 10 10') + .attr('refX', 16) + .attr('refY', 0) + .attr('markerWidth', 4) + .attr('markerHeight', 4) + .attr('orient', 'auto') + .append('svg:path') + .attr('d', 'M0,-5L10,0L0,5') + } + // Links + let link = container.append('svg:g') + .on('click', () => { + renderFooterOnClick(d3.select(d3.event.target).datum(), 'edge') + }) + .selectAll('g.link') + .data(self.force.links()) + .enter() + .append('g') + let getPathId = (d) => this.containerId + '_' + d.source.index + '_' + d.target.index + '_' + d.count + let showLabel = (d) => this._showNodeLabel(d) + let linkPath = link.append('svg:path') + .attr('class', 'link') + .attr('size', linkSize) + .attr('marker-end', `url(#arrowMarker-${this.containerId})`) + let textPath = link.append('svg:path') + .attr('id', getPathId) + .attr('class', 'textpath') + container.append('svg:g') + .selectAll('.pathLabel') + .data(self.force.links()) + .enter() + .append('svg:text') + .attr('class', 'pathLabel') + .append('svg:textPath') + .attr('startOffset', '50%') + .attr('text-anchor', 'middle') + .attr('xlink:href', (d) => '#' + getPathId(d)) + .text((d) => d.label) + .style('opacity', defaultOpacity) + // Nodes + let circle = container.append('svg:g') + .on('click', () => { + renderFooterOnClick(d3.select(d3.event.target).datum(), 'node') + }) + .selectAll('circle') + .data(self.force.nodes()) + .enter().append('svg:circle') + .attr('r', (d) => nodeSize) + .attr('fill', (d) => networkData.graph.labels && d.label in networkData.graph.labels + ? networkData.graph.labels[d.label] : '#000000') + .call(drag) + let text = container.append('svg:g').selectAll('g') + .data(self.force.nodes()) + .enter().append('svg:g') + text.append('svg:text') + .attr('x', (d) => nodeSize + textOffset) + .attr('size', nodeSize) + .attr('y', '.31em') + .attr('class', (d) => 'nodeLabel shadow label-' + d.label) + .text(showLabel) + .style('opacity', defaultOpacity) + text.append('svg:text') + .attr('x', (d) => nodeSize + textOffset) + .attr('size', nodeSize) + .attr('y', '.31em') + .attr('class', (d) => 'nodeLabel label-' + d.label) + .text(showLabel) + .style('opacity', defaultOpacity) + } + + destroy() { + } + + _showNodeLabel(d) { + let transformationConfig = this.transformation.getSetting().scope.config + let selectedLabel = (transformationConfig.properties[d.label] || {selected: 'label'}).selected + return d.data[selectedLabel] || d[selectedLabel] + } + + getTransformation() { + return this.transformation + } +} diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-displayXAxis.html b/zeppelin-web/src/app/visualization/builtins/visualization-displayXAxis.html new file mode 100644 index 0000000000..5a022a04d9 --- /dev/null +++ b/zeppelin-web/src/app/visualization/builtins/visualization-displayXAxis.html @@ -0,0 +1,52 @@ + + +
    + xAxis : +
    + +
    + + + + + +   + + + degree : + + +
    diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-linechart.js b/zeppelin-web/src/app/visualization/builtins/visualization-linechart.js index 31da47b931..959efc8bd9 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-linechart.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-linechart.js @@ -23,6 +23,12 @@ export default class LinechartVisualization extends Nvd3ChartVisualization { super(targetEl, config) this.pivot = new PivotTransformation(config) + + try { + this.config.rotate = {degree: config.rotate.degree} + } catch (e) { + this.config.rotate = {degree: '-45'} + } } type () { @@ -50,6 +56,7 @@ export default class LinechartVisualization extends Nvd3ChartVisualization { this.xLabels = d3Data.xLabels super.render(d3Data) + this.config.changeXLabel(this.config.xLabelStatus) } /** @@ -68,6 +75,8 @@ export default class LinechartVisualization extends Nvd3ChartVisualization { configureChart (chart) { let self = this + let configObj = self.config + chart.xAxis.tickFormat(function (d) { return self.xAxisTickFormat(d, self.xLabels) }) chart.yAxis.tickFormat(function (d) { if (d === undefined) { @@ -84,12 +93,60 @@ export default class LinechartVisualization extends Nvd3ChartVisualization { } else { chart.forceY([]) } + + self.config.changeXLabel = function(type) { + switch (type) { + case 'default': + self.chart._options['showXAxis'] = true + self.chart._options['margin'] = {bottom: 50} + self.chart.xAxis.rotateLabels(0) + configObj.xLabelStatus = 'default' + break + case 'rotate': + self.chart._options['showXAxis'] = true + self.chart._options['margin'] = {bottom: 140} + self.chart.xAxis.rotateLabels(configObj.rotate.degree) + configObj.xLabelStatus = 'rotate' + break + case 'hide': + self.chart._options['showXAxis'] = false + self.chart._options['margin'] = {bottom: 50} + d3.select('#' + self.targetEl[0].id + '> svg').select('g.nv-axis.nv-x').selectAll('*').remove() + configObj.xLabelStatus = 'hide' + break + } + self.emitConfig(configObj) + } + + self.config.isXLabelStatus = function(type) { + if (configObj.xLabelStatus === type) { + return true + } else { + return false + } + } + + self.config.setDegree = function(type) { + configObj.rotate.degree = type + self.chart.xAxis.rotateLabels(type) + self.emitConfig(configObj) + } } getSetting (chart) { let self = this let configObj = self.config + // default to visualize xLabel + if (typeof (configObj.xLabelStatus) === 'undefined') { + configObj.changeXLabel('default') + } + + if (typeof (configObj.rotate.degree) === 'undefined' || configObj.rotate.degree === '') { + configObj.rotate.degree = '-45' + self.emitConfig(configObj) + } + return { template: `
    -
    `, +
    + + `, scope: { config: configObj, save: function () { diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-table-grid-filter.html b/zeppelin-web/src/app/visualization/builtins/visualization-table-grid-filter.html new file mode 100644 index 0000000000..d4b7e987f5 --- /dev/null +++ b/zeppelin-web/src/app/visualization/builtins/visualization-table-grid-filter.html @@ -0,0 +1,27 @@ + + +
    +
    + +
    +
    diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-table-setting.html b/zeppelin-web/src/app/visualization/builtins/visualization-table-setting.html new file mode 100644 index 0000000000..d01fd1b928 --- /dev/null +++ b/zeppelin-web/src/app/visualization/builtins/visualization-table-setting.html @@ -0,0 +1,84 @@ + + +
    +
    + Table Options + +
    +
    + +
    +
    +
    +
    +
    + +
    + + + + + + + + + + + + +
    NameValue
    + + {{optSpec.name}} + + + +
    + +
    +
    + +
    + +
    + +
    + +
    + +
    + +
    +
    +
    diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-table.js b/zeppelin-web/src/app/visualization/builtins/visualization-table.js index 3192ee61f4..f8c280a3f9 100644 --- a/zeppelin-web/src/app/visualization/builtins/visualization-table.js +++ b/zeppelin-web/src/app/visualization/builtins/visualization-table.js @@ -14,7 +14,41 @@ import Visualization from '../visualization' import PassthroughTransformation from '../../tabledata/passthrough' -import HandsonHelper from '../../handsontable/handsonHelper' + +import { + Widget, ValueType, + isInputWidget, isOptionWidget, isCheckboxWidget, + isTextareaWidget, isBtnGroupWidget, + initializeTableConfig, resetTableOptionConfig, + DefaultTableColumnType, TableColumnType, updateColumnTypeState, + parseTableOption, +} from './visualization-util' + +const SETTING_TEMPLATE = require('./visualization-table-setting.html') + +const TABLE_OPTION_SPECS = [ + { + name: 'useFilter', + valueType: ValueType.BOOLEAN, + defaultValue: false, + widget: Widget.CHECKBOX, + description: 'Enable filter for columns', + }, + { + name: 'showPagination', + valueType: ValueType.BOOLEAN, + defaultValue: false, + widget: Widget.CHECKBOX, + description: 'Enable pagination for better navigation', + }, + { + name: 'showAggregationFooter', + valueType: ValueType.BOOLEAN, + defaultValue: false, + widget: Widget.CHECKBOX, + description: 'Enable a footer for displaying aggregated values', + }, +] /** * Visualize data in table format @@ -22,42 +56,349 @@ import HandsonHelper from '../../handsontable/handsonHelper' export default class TableVisualization extends Visualization { constructor (targetEl, config) { super(targetEl, config) - console.log('Init table viz') - targetEl.addClass('table') this.passthrough = new PassthroughTransformation(config) + this.emitTimeout = null + this.isRestoring = false + + initializeTableConfig(config, TABLE_OPTION_SPECS) } - refresh () { - this.hot.render() + createGridOptions(tableData, onRegisterApiCallback, config) { + const rows = tableData.rows + const columnNames = tableData.columns.map(c => c.name) + + const gridData = rows.map(r => { + return columnNames.reduce((acc, colName, index) => { + acc[colName] = r[index] + return acc + }, {}) + }) + + const gridOptions = { + data: gridData, + enableGridMenu: true, + modifierKeysToMultiSelectCells: true, + exporterMenuCsv: true, + exporterMenuPdf: false, + flatEntityAccess: true, + fastWatch: false, + treeRowHeaderAlwaysVisible: false, + + columnDefs: columnNames.map(colName => { + return { + name: colName, + type: DefaultTableColumnType, + cellTemplate: ` +
    + {{grid.getCellValue(row, col)}} +
    +
    +
    + `, + } + }), + rowEditWaitInterval: -1, /** disable saveRow event */ + enableRowHashing: true, + saveFocus: false, + saveScroll: false, + saveSort: true, + savePinning: true, + saveGrouping: true, + saveGroupingExpandedStates: true, + saveOrder: true, // column order + saveVisible: true, // column visibility + saveTreeView: true, + saveFilter: true, + saveSelection: false, + } + + return gridOptions + } + + getGridElemId() { + // angular doesn't allow `-` in scope variable name + const gridElemId = `${this.targetEl[0].id}_grid`.replace('-', '_') + return gridElemId + } + + getGridApiId() { + // angular doesn't allow `-` in scope variable name + const gridApiId = `${this.targetEl[0].id}_gridApi`.replace('-', '_') + return gridApiId + } + + refresh() { + const gridElemId = this.getGridElemId() + const gridElem = angular.element(`#${gridElemId}`) + + if (gridElem) { + gridElem.css('height', this.targetEl.height() - 10) + } + } + + refreshGrid() { + const gridElemId = this.getGridElemId() + const gridElem = angular.element(`#${gridElemId}`) + + if (gridElem) { + const scope = this.getScope() + const gridApiId = this.getGridApiId() + scope[gridApiId].core.notifyDataChange(this._uiGridConstants.dataChange.ALL) + } + } + + updateColDefType(colDef, type) { + if (type === colDef.type) { return } + + colDef.type = type + const colName = colDef.name + const config = this.config + if (config.tableColumnTypeState.names && config.tableColumnTypeState.names[colName]) { + config.tableColumnTypeState.names[colName] = type + this.persistConfigWithGridState(this.config) + } + } + + addColumnMenus(gridOptions) { + if (!gridOptions || !gridOptions.columnDefs) { return } + + const self = this // for closure + + // SHOULD use `function() { ... }` syntax for each action to get `this` + gridOptions.columnDefs.map(colDef => { + colDef.menuItems = [ + { + title: 'Type: String', + action: function() { + self.updateColDefType(this.context.col.colDef, TableColumnType.STRING) + }, + active: function() { + return this.context.col.colDef.type === TableColumnType.STRING + }, + }, + { + title: 'Type: Number', + action: function() { + self.updateColDefType(this.context.col.colDef, TableColumnType.NUMBER) + }, + active: function() { + return this.context.col.colDef.type === TableColumnType.NUMBER + }, + }, + { + title: 'Type: Date', + action: function() { + self.updateColDefType(this.context.col.colDef, TableColumnType.DATE) + }, + active: function() { + return this.context.col.colDef.type === TableColumnType.DATE + }, + }, + ] + }) + } + + setDynamicGridOptions(gridOptions, config) { + // parse based on their type definitions + const parsed = parseTableOption(TABLE_OPTION_SPECS, config.tableOptionValue) + + const { showAggregationFooter, useFilter, showPagination, } = parsed + + gridOptions.showGridFooter = false + gridOptions.showColumnFooter = showAggregationFooter + gridOptions.enableFiltering = useFilter + + gridOptions.enablePagination = showPagination + gridOptions.enablePaginationControls = showPagination + + if (showPagination) { + gridOptions.paginationPageSize = 50 + gridOptions.paginationPageSizes = [25, 50, 100, 250, 1000] + } + + // selection can't be rendered dynamically in ui-grid 4.0.4 + gridOptions.enableRowSelection = false + gridOptions.enableRowHeaderSelection = false + gridOptions.enableFullRowSelection = false + gridOptions.enableSelectAll = false + gridOptions.enableGroupHeaderSelection = false + gridOptions.enableSelectionBatchEvent = false } render (tableData) { - let height = this.targetEl.height() - let container = this.targetEl.css('height', height).get(0) - let resultRows = tableData.rows - let columnNames = _.pluck(tableData.columns, 'name') - // eslint-disable-next-line prefer-spread - let columns = Array.apply(null, Array(tableData.columns.length)).map(function () { - return {type: 'text'} - }) + const gridElemId = this.getGridElemId() + let gridElem = document.getElementById(gridElemId) - if (this.hot) { - this.hot.destroy() + const config = this.config + const self = this // for closure + + if (!gridElem) { + // create, compile and append grid elem + gridElem = angular.element( + `
    `) + + gridElem.css('height', this.targetEl.height() - 10) + const scope = this.getScope() + gridElem = this._compile(gridElem)(scope) + this.targetEl.append(gridElem) + + // set gridOptions for this elem + const gridOptions = this.createGridOptions(tableData, onRegisterApiCallback, config) + this.setDynamicGridOptions(gridOptions, config) + this.addColumnMenus(gridOptions) + scope[gridElemId] = gridOptions + + // set gridApi for this elem + const gridApiId = this.getGridApiId() + const onRegisterApiCallback = (gridApi) => { + scope[gridApiId] = gridApi + // should restore state before registering APIs + + // register callbacks for change evens + // should persist `self.config` instead `config` (closure issue) + gridApi.core.on.columnVisibilityChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + gridApi.colMovable.on.columnPositionChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + gridApi.core.on.sortChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + gridApi.core.on.filterChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + gridApi.grouping.on.aggregationChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + gridApi.grouping.on.groupingChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + gridApi.treeBase.on.rowCollapsed(scope, () => { self.persistConfigWithGridState(self.config) }) + gridApi.treeBase.on.rowExpanded(scope, () => { self.persistConfigWithGridState(self.config) }) + + // pagination doesn't follow usual life-cycle in ui-grid v4.0.4 + // gridApi.pagination.on.paginationChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + // TBD: do we need to propagate row selection? + // gridApi.selection.on.rowSelectionChanged(scope, () => { self.persistConfigWithGridState(self.config) }) + // gridApi.selection.on.rowSelectionChangedBatch(scope, () => { self.persistConfigWithGridState(self.config) }) + } + gridOptions.onRegisterApi = onRegisterApiCallback + } else { + // don't need to update gridOptions.data since it's synchronized by paragraph execution + const gridOptions = this.getGridOptions() + this.setDynamicGridOptions(gridOptions, config) + this.refreshGrid() } - let handsonHelper = new HandsonHelper() - this.hot = new Handsontable(container, handsonHelper.getHandsonTableConfig( - columns, columnNames, resultRows)) - this.hot.validateCells(null) + const columnDefs = this.getGridOptions().columnDefs + updateColumnTypeState(tableData.columns, config, columnDefs) + // SHOULD restore grid state after columnDefs are updated + this.restoreGridState(config.tableGridState) + } + + restoreGridState(gridState) { + if (!gridState) { return } + + // should set isRestoring to avoid that changed* events are triggered while restoring + this.isRestoring = true + const gridApi = this.getGridApi() + + // restore grid state when gridApi is available + if (!gridApi) { + setTimeout(() => this.restoreGridState(gridState), 100) + } else { + gridApi.saveState.restore(this.getScope(), gridState) + this.isRestoring = false + } } destroy () { - if (this.hot) { - this.hot.destroy() - } } getTransformation () { return this.passthrough } + + getScope() { + const scope = this.targetEl.scope() + return scope + } + + getGridOptions() { + const scope = this.getScope() + const gridElemId = this.getGridElemId() + return scope[gridElemId] + } + + getGridApi() { + const scope = this.targetEl.scope() + const gridApiId = this.getGridApiId() + return scope[gridApiId] + } + + persistConfigImmediatelyWithGridState(config) { + this.persistConfigWithGridState(config) + } + + persistConfigWithGridState(config) { + if (this.isRestoring) { return } + + const gridApi = this.getGridApi() + config.tableGridState = gridApi.saveState.save() + this.emitConfig(config) + } + + persistConfig(config) { + this.emitConfig(config) + } + + getSetting (chart) { + const self = this // for closure in scope + const configObj = self.config + + // emit config if it's updated in `render` + if (configObj.initialized) { + configObj.initialized = false + this.persistConfig(configObj) // should persist w/o state + } else if (configObj.tableColumnTypeState && + configObj.tableColumnTypeState.updated) { + configObj.tableColumnTypeState.updated = false + this.persistConfig(configObj) // should persist w/o state + } + + return { + template: SETTING_TEMPLATE, + scope: { + config: configObj, + tableOptionSpecs: TABLE_OPTION_SPECS, + isInputWidget: isInputWidget, + isOptionWidget: isOptionWidget, + isCheckboxWidget: isCheckboxWidget, + isTextareaWidget: isTextareaWidget, + isBtnGroupWidget: isBtnGroupWidget, + tableOptionValueChanged: () => { + self.persistConfigWithGridState(configObj) + }, + saveTableOption: () => { + self.persistConfigWithGridState(configObj) + }, + resetTableOption: () => { + resetTableOptionConfig(configObj) + initializeTableConfig(configObj, TABLE_OPTION_SPECS) + self.persistConfigWithGridState(configObj) + }, + tableWidgetOnKeyDown: (event, optSpec) => { + const code = event.keyCode || event.which + if (code === 13 && isInputWidget(optSpec)) { + self.persistConfigWithGridState(configObj) + } else if (code === 13 && event.shiftKey && isTextareaWidget(optSpec)) { + self.persistConfigWithGridState(configObj) + } + + event.stopPropagation() /** avoid to conflict with paragraph shortcuts */ + } + } + } + } } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-util.js b/zeppelin-web/src/app/visualization/builtins/visualization-util.js new file mode 100644 index 0000000000..cd9cd48b75 --- /dev/null +++ b/zeppelin-web/src/app/visualization/builtins/visualization-util.js @@ -0,0 +1,172 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const Widget = { + CHECKBOX: 'checkbox', + INPUT: 'input', + TEXTAREA: 'textarea', + OPTION: 'option', + BTN_GROUP: 'btn-group', +} + +export const ValueType = { + INT: 'int', + FLOAT: 'float', + BOOLEAN: 'boolean', + STRING: 'string', + JSON: 'JSON', +} + +export const TableColumnType = { + STRING: 'string', + BOOLEAN: 'boolean', + NUMBER: 'number', + DATE: 'date', + OBJECT: 'object', + NUMBER_STR: 'numberStr', +} + +export const DefaultTableColumnType = TableColumnType.STRING + +export function isInputWidget (spec) { return spec.widget === Widget.INPUT } +export function isOptionWidget (spec) { return spec.widget === Widget.OPTION } +export function isCheckboxWidget (spec) { return spec.widget === Widget.CHECKBOX } +export function isTextareaWidget (spec) { return spec.widget === Widget.TEXTAREA } +export function isBtnGroupWidget (spec) { return spec.widget === Widget.BTN_GROUP } + +export function resetTableOptionConfig(config) { + delete config.tableOptionSpecHash + config.tableOptionSpecHash = {} + delete config.tableOptionValue + config.tableOptionValue = {} + delete config.tableColumnTypeState.names + config.tableColumnTypeState.names = {} + config.updated = false + return config +} + +export function initializeTableConfig(config, tableOptionSpecs) { + if (typeof config.tableOptionValue === 'undefined') { config.tableOptionValue = {} } + if (typeof config.tableGridState === 'undefined') { config.tableGridState = {} } + if (typeof config.tableColumnTypeState === 'undefined') { config.tableColumnTypeState = {} } + + // should remove `$$hashKey` using angular.toJson + const newSpecHash = JSON.stringify(JSON.parse(angular.toJson(tableOptionSpecs))) + const previousSpecHash = config.tableOptionSpecHash + + // check whether spec is updated or not + if (typeof previousSpecHash === 'undefined' || (previousSpecHash !== newSpecHash)) { + resetTableOptionConfig(config) + + config.tableOptionSpecHash = newSpecHash + config.initialized = true + + // reset all persisted option values if spec is updated + for (let i = 0; i < tableOptionSpecs.length; i++) { + const option = tableOptionSpecs[i] + config.tableOptionValue[option.name] = option.defaultValue + } + } + + return config +} + +export function parseTableOption(specs, persistedTableOption) { + /** copy original params */ + const parsed = JSON.parse(JSON.stringify(persistedTableOption)) + + for (let i = 0; i < specs.length; i++) { + const s = specs[i] + const name = s.name + + if (s.valueType === ValueType.INT && + typeof parsed[name] !== 'number') { + try { parsed[name] = parseInt(parsed[name]) } catch (error) { parsed[name] = s.defaultValue } + } else if (s.valueType === ValueType.FLOAT && + typeof parsed[name] !== 'number') { + try { parsed[name] = parseFloat(parsed[name]) } catch (error) { parsed[name] = s.defaultValue } + } else if (s.valueType === ValueType.BOOLEAN) { + if (parsed[name] === 'false') { + parsed[name] = false + } else if (parsed[name] === 'true') { + parsed[name] = true + } else if (typeof parsed[name] !== 'boolean') { + parsed[name] = s.defaultValue + } + } else if (s.valueType === ValueType.JSON) { + if (parsed[name] !== null && typeof parsed[name] !== 'object') { + try { parsed[name] = JSON.parse(parsed[name]) } catch (error) { parsed[name] = s.defaultValue } + } else if (parsed[name] === null) { + parsed[name] = s.defaultValue + } + } + } + + return parsed +} + +export function isColumnNameUpdated(prevColumnNames, newColumnNames) { + if (typeof prevColumnNames === 'undefined') { return true } + + let columnNameUpdated = false + + for (let prevColName in prevColumnNames) { + if (!newColumnNames[prevColName]) { + return true + } + } + + if (!columnNameUpdated) { + for (let newColName in newColumnNames) { + if (!prevColumnNames[newColName]) { + return true + } + } + } + + return false +} + +export function updateColumnTypeState(columns, config, columnDefs) { + const columnTypeState = config.tableColumnTypeState + + if (!columnTypeState) { return } + + // compare objects because order might be changed + const prevColumnNames = columnTypeState.names || {} + const newColumnNames = columns.reduce((acc, c) => { + const prevColumnType = prevColumnNames[c.name] + + // use previous column type if exists + if (prevColumnType) { + acc[c.name] = prevColumnType + } else { + acc[c.name] = DefaultTableColumnType + } + return acc + }, {}) + + let columnNameUpdated = isColumnNameUpdated(prevColumnNames, newColumnNames) + + if (columnNameUpdated) { + columnTypeState.names = newColumnNames + columnTypeState.updated = true + } + + // update `columnDefs[n].type` + for (let i = 0; i < columnDefs.length; i++) { + const colName = columnDefs[i].name + columnDefs[i].type = columnTypeState.names[colName] + } +} diff --git a/zeppelin-web/src/app/visualization/visualization.js b/zeppelin-web/src/app/visualization/visualization.js index ec8988238f..82704e3a00 100644 --- a/zeppelin-web/src/app/visualization/visualization.js +++ b/zeppelin-web/src/app/visualization/visualization.js @@ -13,7 +13,7 @@ */ /** - * Base class for visualization + * Base class for visualization. */ export default class Visualization { constructor (targetEl, config) { @@ -25,17 +25,22 @@ export default class Visualization { } /** - * get transformation + * Get transformation. + * @abstract + * @return {Transformation} */ getTransformation () { // override this + throw new TypeError('Visualization.getTransformation() should be overrided') } /** - * Method will be invoked when data or configuration changed + * Method will be invoked when data or configuration changed. + * @abstract */ render (tableData) { // override this + throw new TypeError('Visualization.render() should be overrided') } /** @@ -46,7 +51,7 @@ export default class Visualization { } /** - * method will be invoked when visualization need to be destroyed. + * Method will be invoked when visualization need to be destroyed. * Don't need to destroy this.targetEl. */ destroy () { @@ -64,7 +69,7 @@ export default class Visualization { } /** - * Activate. invoked when visualization is selected + * Activate. Invoked when visualization is selected. */ activate () { if (!this._active || this._dirty) { @@ -75,21 +80,21 @@ export default class Visualization { } /** - * Activate. invoked when visualization is de selected + * Deactivate. Invoked when visualization is de selected. */ deactivate () { this._active = false } /** - * Is active + * Is active. */ isActive () { return this._active } /** - * When window or paragraph is resized + * When window or paragraph is resized. */ resize () { if (this.isActive()) { @@ -100,7 +105,7 @@ export default class Visualization { } /** - * Set new config + * Set new config. */ setConfig (config) { this.config = config @@ -119,7 +124,7 @@ export default class Visualization { } /** - * render setting + * Render setting. */ renderSetting (targetEl) { let setting = this.getSetting() diff --git a/zeppelin-web/src/components/baseUrl/baseUrl.service.js b/zeppelin-web/src/components/baseUrl/baseUrl.service.js index fb2ecfbfd8..5b8824a098 100644 --- a/zeppelin-web/src/components/baseUrl/baseUrl.service.js +++ b/zeppelin-web/src/components/baseUrl/baseUrl.service.js @@ -36,10 +36,12 @@ function baseUrlSrv () { skipTrailingSlash(location.pathname) + '/ws' } + this.getBase = function() { + return location.protocol + '//' + location.hostname + ':' + this.getPort() + location.pathname + } + this.getRestApiBase = function () { - return location.protocol + '//' + location.hostname + ':' + - this.getPort() + skipTrailingSlash(location.pathname) + - '/api' + return skipTrailingSlash(this.getBase()) + '/api' } const skipTrailingSlash = function (path) { diff --git a/zeppelin-web/src/components/editor/ace.editor.directive.html b/zeppelin-web/src/components/editor/ace.editor.directive.html index 4729e4bb28..1c0a1f0f48 100644 --- a/zeppelin-web/src/components/editor/ace.editor.directive.html +++ b/zeppelin-web/src/components/editor/ace.editor.directive.html @@ -13,10 +13,7 @@ limitations under the License. -->
    diff --git a/zeppelin-web/src/components/editor/codeEditor.directive.js b/zeppelin-web/src/components/editor/codeEditor.directive.js index b8e1b6a503..d8cb73f025 100644 --- a/zeppelin-web/src/components/editor/codeEditor.directive.js +++ b/zeppelin-web/src/components/editor/codeEditor.directive.js @@ -31,7 +31,7 @@ function codeEditor ($templateRequest, $compile) { editor.attr('id', scope.paragraphId + '_editor') element.append(editor) $compile(editor)(scope) - console.log('codeEditor directive revision view is ' + scope.revisionView) + console.debug('codeEditor directive revision view is ' + scope.revisionView) }) } } diff --git a/zeppelin-web/src/components/expandCollapse/expandCollapse.directive.js b/zeppelin-web/src/components/expandCollapse/expandCollapse.directive.js index 5d51818b6e..c71fae0c87 100644 --- a/zeppelin-web/src/components/expandCollapse/expandCollapse.directive.js +++ b/zeppelin-web/src/components/expandCollapse/expandCollapse.directive.js @@ -21,12 +21,12 @@ function expandCollapse () { angular.element(element).click(function (event) { if (angular.element(element).find('.expandable:visible').length > 1) { angular.element(element).find('.expandable:visible').slideUp('slow') - angular.element(element).find('i.icon-folder-alt').toggleClass('icon-folder icon-folder-alt') + angular.element(element).find('i.fa-folder-open').toggleClass('fa-folder fa-folder-open') } else { angular.element(element).find('.expandable').first().slideToggle('200', function () { // do not toggle trash folder if (angular.element(element).find('.fa-trash-o').length === 0) { - angular.element(element).find('i').first().toggleClass('icon-folder icon-folder-alt') + angular.element(element).find('i').first().toggleClass('fa-folder fa-folder-open') } }) } diff --git a/zeppelin-web/src/components/navbar/navbar-noteList-elem.html b/zeppelin-web/src/components/navbar/navbar-noteList-elem.html index d5fb4a13b5..6cba04a987 100644 --- a/zeppelin-web/src/components/navbar/navbar-noteList-elem.html +++ b/zeppelin-web/src/components/navbar/navbar-noteList-elem.html @@ -22,7 +22,7 @@ limitations under the License.
    - + {{noteName(node)}} span { + font-family: 'Patua One', cursive; + font-size: 25px; + color: white; +} + +.navbar-menu { + font-weight: 300; + font-size: 16px; +} + +.navbar-inverse .navbar-nav > li > a.navbar-menu-notebook:focus { + background-color: transparent; +} + +.navbar-inverse .navbar-nav > li > a.navbar-menu-notebook:hover{ + background-color: rgba(0, 0, 0, 0.2); +} + +.navbar-inverse .navbar-nav > li > a { + padding-top: 16px; +} + +.navbar-logo { + display: inline-block; + padding-right: 10px; +} + +.navbar-search { + font-size: 14px; + font-family: 'FontAwesome', 'Helvetica Neue', Helvetica, Arial, sans-serif; +} + +#navbar-search { + min-width: 170px; +} @media (min-width: 795px) and (max-width: 830px) { - input#searchTermId { - width: 170px; - min-width: 170px; + #navbar-search { + width: 150px; + min-width: 150px; } } @media (min-width: 768px) and (max-width: 794px) { - input#searchTermId { - width: 140px; - min-width: 140px; + #navbar-search { + width: 130px; + min-width: 130px; } } @@ -41,3 +82,151 @@ } .navbar-fixed-top.headroom--unpinned { top: -100px; } .navbar-fixed-top.headroom--pinned { top: 0; /** `navbar` top */ } + +.navbar-inverse { + background: #3071a9; + color: #fff; + border-color: #3071a9; + font-size: 18px; +} + +.navbar-inverse .navbar-nav > .open > a, +.navbar-inverse .navbar-nav > .open > a:hover, +.navbar-inverse .navbar-nav > .open > a:focus { + color: #fff; + background: #3071a9; +} + +.navbar-inverse .navbar-toggle:hover, +.navbar-inverse .navbar-toggle:focus { + background: #3071a9; +} + +.navbar-inverse .navbar-nav > li > a:hover, +.navbar-inverse .navbar-nav > li > a:focus { + color: #fff; + background: rgba(0, 0, 0, 0.2); +} + +.navbar-inverse .navbar-toggle { + border-color: transparent; +} + +.navbar-inverse .navbar-nav > .active > a, +.navbar-inverse .navbar-nav > .active > a:hover, +.navbar-inverse .navbar-nav > .active > a:focus { + color: #fff; + background: #080808; +} + +.navbar-inverse .navbar-nav > li > a { + color: #FFFFFF; +} + +.navbar-inverse .navbar-brand { + color: #fff; + text-decoration: none; + font-family: 'Patua One', cursive; + font-size: 32px; +} + +a.navbar-brand:hover { + color: #fff !important; +} + +@media (max-width: 767px) { + .navbar-inverse .navbar-nav .open .dropdown-menu > li > a, + #notebook-list li a { + color: #ecf0f1; + } + + .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a, + #notebook-list li a { + padding: 5px 15px 5px 25px; + line-height: 20px; + } + + .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a { + color: #D3D3D3; + } + + .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a:hover, + .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > li > a:focus, + #notebook-list li a:hover { + color: #fff; + background: transparent; + } + + .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a, + .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a:hover, + .navbar-inverse .navbar-nav .open .dropdown-menu > .scrollbar-container > .active > a:focus { + color: #fff; + background: #080808; + } + + .navbar-inverse .navbar-nav .open .dropdown-menu .divider { + background: #3071A9; + } + + .navbar-inverse .navbar-collapse, .navbar-inverse .navbar-form { + border-color: #3071A9; + } +} + +.username { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 120px; + display: inline-block; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; +} + + +/* About Dialog */ + +.modal-header-about { + background-color: #3071a9; + border: 2px solid #3071a9; + border-top-left-radius: 4px; + border-top-right-radius: 4px; +} + +.modal-header-about > .close { + color: #cfcfcf; + opacity: 1; +} + +.modal-header-about > .modal-title { + font-weight: 300; + font-size: 20px; + color: #e8e8e8; +} + +.modal-body-about-version { + font-weight: 500; +} + +.modal-body-about { + height: 250px; + padding: 50px; +} + +.modal-body-about .logo { + padding-top: 30px; +} + +.modal-body-about .logo img { + width: 95%; +} + +.modal-body-about .content { + text-align: center; +} + +.modal-body-about .content h3 { + font-family: 'Patua One'; + color: #3071A9; + font-size: 30px; + margin: 0 auto; +} diff --git a/zeppelin-web/src/components/navbar/navbar.html b/zeppelin-web/src/components/navbar/navbar.html index 8c6749288e..d5e6668fa9 100644 --- a/zeppelin-web/src/components/navbar/navbar.html +++ b/zeppelin-web/src/components/navbar/navbar.html @@ -20,18 +20,30 @@ limitations under the License. - - I'm zeppelin Zeppelin + + + Zeppelin
    -