Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[ZEPPELIN-2297] schema filters

Browse source
This commit is contained in:
Tinkoff DWH 2017-03-28 18:12:50 +05:00
parent 5308f1e651
commit 8552049020
5 changed files with 69 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
docs/interpreter/jdbc.md
View file

@ -123,6 +123,11 @@ The JDBC interpreter properties are defined by default like below.
<td></td>
<td>Some SQL which executes while opening connection</td>
</tr>
<tr>
<td>default.completer.schemaFilters</td>
<td></td>
<td>Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)</td>
</tr>
</table>
If you want to connect other databases such as `Mysql`, `Redshift` and `Hive`, you need to edit the property values.

10
jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
View file

@ -100,6 +100,7 @@ public class JDBCInterpreter extends Interpreter {
static final String USER_KEY = "user";
static final String PASSWORD_KEY = "password";
static final String PRECODE_KEY = "precode";
static final String COMPLETER_SCHEMA_FILTERS_KEY = "completer.schemaFilters";
static final String JDBC_JCEKS_FILE = "jceks.file";
static final String JDBC_JCEKS_CREDENTIAL_KEY = "jceks.credentialKey";
static final String PRECODE_KEY_TEMPLATE = "%s.precode";
@ -191,10 +192,11 @@ public class JDBCInterpreter extends Interpreter {
}
}
private SqlCompleter createSqlCompleter(Connection jdbcConnection) {
private SqlCompleter createSqlCompleter(Connection jdbcConnection, String propertyKey) {
String schemaFiltersKey = String.format("%s.%s", propertyKey, COMPLETER_SCHEMA_FILTERS_KEY);
String filters = getProperty(schemaFiltersKey);
SqlCompleter completer = new SqlCompleter();
completer.initFromConnection(jdbcConnection, "");
completer.initFromConnection(jdbcConnection, filters);
return completer;
}
@ -754,7 +756,7 @@ public class JDBCInterpreter extends Interpreter {
logger.warn("SQLCompleter will created without use connection");
}
SqlCompleter sqlCompleter = createSqlCompleter(connection);
SqlCompleter sqlCompleter = createSqlCompleter(connection, propertyKey);
if (sqlCompleter != null) {
sqlCompleter.complete(buf, cursor - 1, candidates);

86
jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
View file

@ -12,6 +12,7 @@ import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -86,7 +87,7 @@ public class SqlCompleter {
Pattern whitespaceEndPatter = Pattern.compile("\\s$");
String cursorArgument = null;
int argumentPosition = 0;
int argumentPosition;
if (buffer.length() == 0 || whitespaceEndPatter.matcher(buffer).find()) {
argumentPosition = buffer.length() - 1;
} else {
@ -114,7 +115,6 @@ public class SqlCompleter {
candidates.set(0, interpreterCompletion);
}
logger.debug("complete:" + complete + ", size:" + candidates.size());
return complete;
}
@ -122,24 +122,26 @@ public class SqlCompleter {
* Return list of schema names within the database
*
* @param meta metadata from connection to database
* @param schemaFilter a schema name pattern; must match the schema name
* @param schemaFilters a schema name patterns; must match the schema name
* as it is stored in the database; "" retrieves those without a schema;
* <code>null</code> means that the schema name should not be used to narrow
* the search; supports '%' and '_' symbols; for example "prod_v_%"
* the search; supports '%'; for example "prod_v_%"
* @return set of all schema names in the database
*/
private static Set<String> getSchemaNames(DatabaseMetaData meta, String schemaFilter) {
private static Set<String> getSchemaNames(DatabaseMetaData meta, List<String> schemaFilters) {
Set<String> res = new HashSet<>();
try {
ResultSet schemas = meta.getSchemas();
try {
while (schemas.next()) {
String schemaName = schemas.getString("TABLE_SCHEM");
if (schemaName == null)
if (schemaName == null) {
schemaName = "";
if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
schemaFilter.replace("_", ".").replace("%", ".*?"))) {
res.add(schemaName);
}
for (String schemaFilter : schemaFilters) {
if (schemaFilter.equals("") || schemaName.matches(schemaFilter.replace("%", ".*?"))) {
res.add(schemaName);
}
}
}
} finally {
@ -155,22 +157,23 @@ public class SqlCompleter {
* Return list of catalog names within the database
*
* @param meta metadata from connection to database
* @param schemaFilter a catalog name pattern; must match the catalog name
* @param schemaFilters a catalog name patterns; must match the catalog name
* as it is stored in the database; "" retrieves those without a catalog;
* <code>null</code> means that the schema name should not be used to narrow
* the search; supports '%' and '_' symbols; for example "prod_v_%"
* the search; supports '%'; for example "prod_v_%"
* @return set of all catalog names in the database
*/
private static Set<String> getCatalogNames(DatabaseMetaData meta, String schemaFilter) {
private static Set<String> getCatalogNames(DatabaseMetaData meta, List<String> schemaFilters) {
Set<String> res = new HashSet<>();
try {
ResultSet schemas = meta.getCatalogs();
try {
while (schemas.next()) {
String schemaName = schemas.getString("TABLE_CAT");
if (schemaFilter.equals("") || schemaFilter == null || schemaName.matches(
schemaFilter.replace("_", ".").replace("%", ".*?"))) {
res.add(schemaName);
for (String schemaFilter : schemaFilters) {
if (schemaFilter.equals("") || schemaName.matches(schemaFilter.replace("%", ".*?"))) {
res.add(schemaName);
}
}
}
} finally {
@ -190,7 +193,7 @@ public class SqlCompleter {
* @param schemaFilter a schema name pattern; must match the schema name
* as it is stored in the database; "" retrieves those without a schema;
* <code>null</code> means that the schema name should not be used to narrow
* the search; supports '%' and '_' symbols; for example "prod_v_%"
* the search; supports '%'; for example "prod_v_%"
* @param tables function fills this map, for every schema name adds
* set of table names within the schema
* @param columns function fills this map, for every table name adds set
@ -201,18 +204,27 @@ public class SqlCompleter {
Map<String, Set<String>> tables,
Map<String, Set<String>> columns) {
try {
ResultSet cols = meta.getColumns(catalogName, schemaFilter, "%", "%");
ResultSet cols = meta.getColumns(catalogName, StringUtils.EMPTY, "%", "%");
try {
while (cols.next()) {
String schema = cols.getString("TABLE_SCHEM");
if (schema == null) schema = cols.getString("TABLE_CAT");
if (schema == null) {
schema = cols.getString("TABLE_CAT");
}
if (!schemaFilter.equals("") && !schema.matches(schemaFilter.replace("%", ".*?"))) {
continue;
}
String table = cols.getString("TABLE_NAME");
String column = cols.getString("COLUMN_NAME");
if (!isBlank(table)) {
String schemaTable = schema + "." + table;
if (!columns.containsKey(schemaTable)) columns.put(schemaTable, new HashSet<String>());
if (!columns.containsKey(schemaTable)) {
columns.put(schemaTable, new HashSet<String>());
}
columns.get(schemaTable).add(column);
if (!tables.containsKey(schema)) tables.put(schema, new HashSet<String>());
if (!tables.containsKey(schema)) {
tables.put(schema, new HashSet<String>());
}
tables.get(schema).add(table);
}
}
@ -350,12 +362,14 @@ public class SqlCompleter {
* Initializes all local completers from database connection
*
* @param connection database connection
* @param schemaFilter a schema name pattern; must match the schema name
* as it is stored in the database; "" retrieves those without a schema;
* <code>null</code> means that the schema name should not be used to narrow
* the search; supports '%' and '_' symbols; for example "prod_v_%"
* @param schemaFiltersString a comma separated schema name patterns; supports '%' symbol;
* for example "prod_v_%,prod_t_%"
*/
public void initFromConnection(Connection connection, String schemaFilter) {
public void initFromConnection(Connection connection, String schemaFiltersString) {
if (schemaFiltersString == null) {
schemaFiltersString = StringUtils.EMPTY;
}
List<String> schemaFilters = Arrays.asList(schemaFiltersString.split(","));
try (Connection c = connection) {
Map<String, Set<String>> tables = new HashMap<>();
@ -364,19 +378,15 @@ public class SqlCompleter {
Set<String> catalogs = new HashSet<>();
Set<String> keywords = getSqlKeywordsCompletions(connection);
if (connection != null) {
schemas = getSchemaNames(connection.getMetaData(), schemaFilter);
catalogs = getCatalogNames(connection.getMetaData(), schemaFilter);
if (!"".equals(connection.getCatalog())) {
if (schemas.size() == 0 )
schemas.add(connection.getCatalog());
fillTableAndColumnNames(connection.getCatalog(), connection.getMetaData(), schemaFilter,
tables, columns);
} else {
if (schemas.size() == 0) schemas.addAll(catalogs);
for (String catalog : catalogs) {
fillTableAndColumnNames(catalog, connection.getMetaData(), schemaFilter, tables,
columns);
schemas = getSchemaNames(connection.getMetaData(), schemaFilters);
catalogs = getCatalogNames(connection.getMetaData(), schemaFilters);
if (schemas.size() == 0) {
schemas.addAll(catalogs);
}
for (String schema : schemas) {
for (String schemaFilter : schemaFilters) {
fillTableAndColumnNames(schema, connection.getMetaData(), schemaFilter, tables,
columns);
}
}
}

6
jdbc/src/main/resources/interpreter-setting.json
View file

@ -28,6 +28,12 @@
"defaultValue": "org.postgresql.Driver",
"description": "JDBC Driver Name"
},
"default.completer.schemaFilters": {
"envName": null,
"propertyName": "default.completer.schemaFilters",
"defaultValue": "",
"description": "Сomma separated schema (schema = catalog = database) filters to get metadata for completions. Supports '%' symbol is equivalent to any set of characters. (ex. prod_v_%,public%,info)"
},
"default.precode": {
"envName": null,
"propertyName": "zeppelin.jdbc.precode",

8
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -129,15 +129,15 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
var initializeDefault = function(config) {
var forms = $scope.paragraph.settings.forms;
if (!config.colWidth) {
config.colWidth = 12;
}
if (config.enabled === undefined) {
config.enabled = true;
}
for (var idx in forms) {
if (forms[idx]) {
if (forms[idx].options) {
@ -862,7 +862,7 @@ function ParagraphCtrl($scope, $rootScope, $route, $window, $routeParams, $locat
};
var getInterpreterName = function(paragraphText) {
var intpNameRegexp = /^\s*%(.+?)\s/g;
var intpNameRegexp = /^\s*%(.+?)(\s|\()/g;
var match = intpNameRegexp.exec(paragraphText);
if (match) {
return match[1].trim();