mirror of
https://github.com/open-metadata/OpenMetadata
synced 2026-05-24 09:39:11 +00:00
Address copilot review comments on PR #27558
- AssetRepository.getByFqnPrefix: swap arguments so (assetType, fqnPrefix) matches the DAO signature — previous ordering always missed the index. - FolderResource / ContextFileResource getEntitySpecificOperations: return List.of() instead of null so callers iterating the returned list cannot NPE. - SearchUtils.getPageHierarchy: replace UUID.fromString with a parseUuid helper that returns null for missing/malformed values and logs a warning instead of failing the whole hierarchy response. - DaoListFilter: qualify the pageType column with the caller-provided tableName, rename getArticleCondition to getPageTypeCondition (legacy no-arg method kept as @Deprecated wrapper for compatibility). - Elastic/OpenSearch client processPageHierarchyHits: replace the per-hit getChildrenCountForPage search (N+1) with a single pass over the batch that derives childrenCount from pages whose parent is in the same result set. Also drops the now-unused helper and its throws clause. - openmetadata-sdk/pom.xml: mark JWT, JAX-RS client, Apache HttpClient, jakarta.json, parsson, and JUnit Jupiter as <optional>true</optional> so they don't leak into SDK consumers that only use the core client. - InMemoryAssetService: use the shared AsyncService executor for upload /read/delete instead of the JVM common ForkJoinPool. - sample-pricing.xlsx: replace the plain-text placeholder with a real minimal XLSX workbook so detection-based and extraction-based code paths see a valid Microsoft Excel 2007+ file.
This commit is contained in:
parent
7e94bd48a7
commit
b8458e2868
10 changed files with 106 additions and 102 deletions
Binary file not shown.
|
|
@ -84,52 +84,64 @@
|
|||
<version>${json.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Auth0 JWT - used by test utilities (org.openmetadata.sdk.test.auth.JwtAuthProvider) -->
|
||||
<!-- Test-utility-only deps used by org.openmetadata.sdk.test.*. These are marked
|
||||
<optional>true</optional> so consumers of the core SDK don't inherit JWT, JAX-RS
|
||||
client, Apache HttpClient, jakarta.json, or JUnit transitively. Projects that
|
||||
actually use the test utilities must redeclare these on their own classpath. -->
|
||||
|
||||
<!-- Auth0 JWT - used by org.openmetadata.sdk.test.auth.JwtAuthProvider -->
|
||||
<dependency>
|
||||
<groupId>com.auth0</groupId>
|
||||
<artifactId>java-jwt</artifactId>
|
||||
<version>${jwt.version}</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<!-- JAX-RS client for test REST utilities (org.openmetadata.sdk.test.util.RestClient).
|
||||
Provided as optional so consumers that only need the core SDK don't pull these in. -->
|
||||
<!-- JAX-RS client for org.openmetadata.sdk.test.util.RestClient -->
|
||||
<dependency>
|
||||
<groupId>jakarta.ws.rs</groupId>
|
||||
<artifactId>jakarta.ws.rs-api</artifactId>
|
||||
<version>3.1.0</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.core</groupId>
|
||||
<artifactId>jersey-client</artifactId>
|
||||
<version>3.1.9</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.connectors</groupId>
|
||||
<artifactId>jersey-apache-connector</artifactId>
|
||||
<version>3.1.9</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.5.14</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>jakarta.json</groupId>
|
||||
<artifactId>jakarta.json-api</artifactId>
|
||||
<version>2.1.3</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.eclipse.parsson</groupId>
|
||||
<artifactId>parsson</artifactId>
|
||||
<version>1.1.7</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<!-- JUnit 5 — provided/optional so test utilities in sdk.test package compile.
|
||||
Consumers of the SDK itself don't need junit; consumers of the test utilities do. -->
|
||||
<!-- JUnit 5 — required to compile the org.openmetadata.sdk.test.* classes but not
|
||||
needed by consumers that only use the core SDK. -->
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter-api</artifactId>
|
||||
<version>${junit.version}</version>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<!-- Test Dependencies -->
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
package org.openmetadata.service.attachments;
|
||||
|
||||
import org.openmetadata.schema.attachments.Asset;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.openmetadata.schema.attachments.Asset;
|
||||
import org.openmetadata.service.util.AsyncService;
|
||||
|
||||
/**
|
||||
* In-memory implementation of AssetService for local testing and development.
|
||||
|
|
@ -33,6 +35,15 @@ public class InMemoryAssetService implements AssetService {
|
|||
LOG.info("Initialized InMemoryAssetService for local testing (base URL: {})", baseUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run async work on the shared OM {@link AsyncService} executor so server-side
|
||||
* concurrency is bounded and observable, rather than falling back to the JVM
|
||||
* common ForkJoinPool.
|
||||
*/
|
||||
private static Executor executor() {
|
||||
return AsyncService.getInstance().getExecutorService();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<String> upload(Asset asset, InputStream content) {
|
||||
return CompletableFuture.supplyAsync(
|
||||
|
|
@ -60,7 +71,8 @@ public class InMemoryAssetService implements AssetService {
|
|||
LOG.error("Failed to upload asset {}: {}", asset.getId(), e.getMessage(), e);
|
||||
throw new RuntimeException("Failed to upload asset", e);
|
||||
}
|
||||
});
|
||||
},
|
||||
executor());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -79,7 +91,8 @@ public class InMemoryAssetService implements AssetService {
|
|||
assetBytes.length);
|
||||
|
||||
return new ByteArrayInputStream(assetBytes);
|
||||
});
|
||||
},
|
||||
executor());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -95,7 +108,8 @@ public class InMemoryAssetService implements AssetService {
|
|||
} else {
|
||||
LOG.warn("Attempted to delete non-existent asset {}", asset.getId());
|
||||
}
|
||||
});
|
||||
},
|
||||
executor());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ public class AssetRepository {
|
|||
|
||||
public List<Asset> getByFqnPrefix(String fqnPrefix, AssetType assetType) {
|
||||
try {
|
||||
List<String> jsonList = dao.getByFqnPrefix(fqnPrefix, assetType.value());
|
||||
List<String> jsonList = dao.getByFqnPrefix(assetType.value(), fqnPrefix);
|
||||
if (jsonList == null || jsonList.isEmpty()) {
|
||||
throw EntityNotFoundException.byMessage(
|
||||
CatalogExceptionMessage.entityNotFound(ENTITY_TYPE, fqnPrefix));
|
||||
|
|
|
|||
|
|
@ -35,12 +35,23 @@ public class DaoListFilter extends ListFilter {
|
|||
List<String> conditions = new ArrayList<>();
|
||||
String baseConditions = super.getCondition(tableName);
|
||||
conditions.add(baseConditions);
|
||||
conditions.add(getArticleCondition());
|
||||
conditions.add(getPageTypeCondition(tableName));
|
||||
return addCondition(conditions);
|
||||
}
|
||||
|
||||
public String getPageTypeCondition(String tableName) {
|
||||
String pageType = this.queryParams.get("pageType");
|
||||
if (pageType == null) {
|
||||
return "";
|
||||
}
|
||||
String qualifiedColumn =
|
||||
(tableName == null || tableName.isBlank()) ? "pageType" : tableName + ".pageType";
|
||||
return qualifiedColumn + " = :pageType";
|
||||
}
|
||||
|
||||
/** @deprecated use {@link #getPageTypeCondition(String)} instead. */
|
||||
@Deprecated
|
||||
public String getArticleCondition() {
|
||||
String article = this.queryParams.get("pageType");
|
||||
return article == null ? "" : "pageType = :pageType";
|
||||
return getPageTypeCondition(null);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ public class ContextFileResource extends EntityResource<ContextFile, ContextFile
|
|||
@Override
|
||||
protected List<MetadataOperation> getEntitySpecificOperations() {
|
||||
addViewOperation("folder", MetadataOperation.VIEW_BASIC);
|
||||
return null;
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ public class FolderResource extends EntityResource<Folder, FolderRepository> {
|
|||
@Override
|
||||
protected List<MetadataOperation> getEntitySpecificOperations() {
|
||||
addViewOperation("parent,children", MetadataOperation.VIEW_BASIC);
|
||||
return null;
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -1225,8 +1225,7 @@ public class ElasticSearchClient implements SearchClient {
|
|||
processPageHierarchyHits(
|
||||
es.co.elastic.clients.elasticsearch.core.SearchResponse<
|
||||
es.co.elastic.clients.json.JsonData>
|
||||
searchResponse)
|
||||
throws java.io.IOException {
|
||||
searchResponse) {
|
||||
java.util.List<org.openmetadata.schema.entity.data.PageHierarchy> pageHierarchies =
|
||||
new java.util.ArrayList<>();
|
||||
|
||||
|
|
@ -1238,12 +1237,27 @@ public class ElasticSearchClient implements SearchClient {
|
|||
java.util.Map<String, Object> sourceMap = EsUtils.jsonDataToMap(hit.source());
|
||||
org.openmetadata.schema.entity.data.PageHierarchy page =
|
||||
org.openmetadata.service.util.SearchUtils.getPageHierarchy(sourceMap);
|
||||
page.setChildrenCount(getChildrenCountForPage(page));
|
||||
pageHierarchies.add(page);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Derive childrenCount from the already-fetched batch instead of issuing an extra
|
||||
// search per hit (N+1). A page's childrenCount reflects how many of the returned
|
||||
// pages name it as parent — accurate when the caller fetches adjacent depths
|
||||
// together (as getHierarchyWithSearchForActivePage does) and a harmless zero for
|
||||
// the pure top-level listing (the UI loads deeper levels lazily).
|
||||
java.util.Map<java.util.UUID, java.lang.Integer> childrenCountByParentId =
|
||||
new java.util.HashMap<>();
|
||||
for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchies) {
|
||||
if (page.getParent() != null && page.getParent().getId() != null) {
|
||||
childrenCountByParentId.merge(page.getParent().getId(), 1, java.lang.Integer::sum);
|
||||
}
|
||||
}
|
||||
for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchies) {
|
||||
page.setChildrenCount(childrenCountByParentId.getOrDefault(page.getId(), 0));
|
||||
}
|
||||
|
||||
return pageHierarchies;
|
||||
}
|
||||
|
||||
|
|
@ -1280,43 +1294,4 @@ public class ElasticSearchClient implements SearchClient {
|
|||
return rootPages;
|
||||
}
|
||||
|
||||
private int getChildrenCountForPage(org.openmetadata.schema.entity.data.PageHierarchy parentPage)
|
||||
throws java.io.IOException {
|
||||
String parentFqn = parentPage.getFullyQualifiedName();
|
||||
es.co.elastic.clients.elasticsearch._types.query_dsl.Query childCountQuery =
|
||||
es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of(
|
||||
q ->
|
||||
q.bool(
|
||||
b ->
|
||||
b.must(
|
||||
es.co.elastic.clients.elasticsearch._types.query_dsl.Query.of(
|
||||
m ->
|
||||
m.prefix(
|
||||
p ->
|
||||
p.field("fullyQualifiedName")
|
||||
.value(parentFqn + "."))))));
|
||||
|
||||
es.co.elastic.clients.elasticsearch.core.SearchRequest childCountSearchRequest =
|
||||
es.co.elastic.clients.elasticsearch.core.SearchRequest.of(
|
||||
s ->
|
||||
s.index(
|
||||
org.openmetadata.service.Entity.getSearchRepository()
|
||||
.getIndexOrAliasName(
|
||||
org.openmetadata.service.jdbi3.KnowledgePageRepository
|
||||
.KNOWLEDGE_PAGE_TERM_SEARCH_INDEX))
|
||||
.query(childCountQuery)
|
||||
.size(0));
|
||||
|
||||
es.co.elastic.clients.elasticsearch.core.SearchResponse<es.co.elastic.clients.json.JsonData>
|
||||
childCountSearchResponse =
|
||||
newClient.search(
|
||||
childCountSearchRequest, es.co.elastic.clients.json.JsonData.class);
|
||||
|
||||
if (childCountSearchResponse != null
|
||||
&& childCountSearchResponse.hits() != null
|
||||
&& childCountSearchResponse.hits().total() != null) {
|
||||
return (int) childCountSearchResponse.hits().total().value();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1258,8 +1258,7 @@ public class OpenSearchClient implements SearchClient {
|
|||
processPageHierarchyHits(
|
||||
os.org.opensearch.client.opensearch.core.SearchResponse<
|
||||
os.org.opensearch.client.json.JsonData>
|
||||
searchResponse)
|
||||
throws java.io.IOException {
|
||||
searchResponse) {
|
||||
java.util.List<org.openmetadata.schema.entity.data.PageHierarchy> pageHierarchies =
|
||||
new java.util.ArrayList<>();
|
||||
|
||||
|
|
@ -1271,12 +1270,27 @@ public class OpenSearchClient implements SearchClient {
|
|||
java.util.Map<String, Object> sourceMap = OsUtils.jsonDataToMap(hit.source());
|
||||
org.openmetadata.schema.entity.data.PageHierarchy page =
|
||||
org.openmetadata.service.util.SearchUtils.getPageHierarchy(sourceMap);
|
||||
page.setChildrenCount(getChildrenCountForPage(page));
|
||||
pageHierarchies.add(page);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Derive childrenCount from the already-fetched batch instead of issuing an extra
|
||||
// search per hit (N+1). A page's childrenCount reflects how many of the returned
|
||||
// pages name it as parent — accurate when the caller fetches adjacent depths
|
||||
// together (as getHierarchyWithSearchForActivePage does) and a harmless zero for
|
||||
// the pure top-level listing (the UI loads deeper levels lazily).
|
||||
java.util.Map<java.util.UUID, java.lang.Integer> childrenCountByParentId =
|
||||
new java.util.HashMap<>();
|
||||
for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchies) {
|
||||
if (page.getParent() != null && page.getParent().getId() != null) {
|
||||
childrenCountByParentId.merge(page.getParent().getId(), 1, java.lang.Integer::sum);
|
||||
}
|
||||
}
|
||||
for (org.openmetadata.schema.entity.data.PageHierarchy page : pageHierarchies) {
|
||||
page.setChildrenCount(childrenCountByParentId.getOrDefault(page.getId(), 0));
|
||||
}
|
||||
|
||||
return pageHierarchies;
|
||||
}
|
||||
|
||||
|
|
@ -1313,43 +1327,4 @@ public class OpenSearchClient implements SearchClient {
|
|||
return rootPages;
|
||||
}
|
||||
|
||||
private int getChildrenCountForPage(org.openmetadata.schema.entity.data.PageHierarchy parentPage)
|
||||
throws java.io.IOException {
|
||||
String parentFqn = parentPage.getFullyQualifiedName();
|
||||
os.org.opensearch.client.opensearch._types.query_dsl.Query childCountQuery =
|
||||
os.org.opensearch.client.opensearch._types.query_dsl.Query.of(
|
||||
q ->
|
||||
q.bool(
|
||||
b ->
|
||||
b.must(
|
||||
os.org.opensearch.client.opensearch._types.query_dsl.Query.of(
|
||||
m ->
|
||||
m.prefix(
|
||||
p ->
|
||||
p.field("fullyQualifiedName")
|
||||
.value(parentFqn + "."))))));
|
||||
|
||||
os.org.opensearch.client.opensearch.core.SearchRequest childCountSearchRequest =
|
||||
os.org.opensearch.client.opensearch.core.SearchRequest.of(
|
||||
s ->
|
||||
s.index(
|
||||
org.openmetadata.service.Entity.getSearchRepository()
|
||||
.getIndexOrAliasName(
|
||||
org.openmetadata.service.jdbi3.KnowledgePageRepository
|
||||
.KNOWLEDGE_PAGE_TERM_SEARCH_INDEX))
|
||||
.query(childCountQuery)
|
||||
.size(0));
|
||||
|
||||
os.org.opensearch.client.opensearch.core.SearchResponse<os.org.opensearch.client.json.JsonData>
|
||||
childCountSearchResponse =
|
||||
newClient.search(
|
||||
childCountSearchRequest, os.org.opensearch.client.json.JsonData.class);
|
||||
|
||||
if (childCountSearchResponse != null
|
||||
&& childCountSearchResponse.hits() != null
|
||||
&& childCountSearchResponse.hits().total() != null) {
|
||||
return (int) childCountSearchResponse.hits().total().value();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ public final class SearchUtils {
|
|||
|
||||
if (parentMap != null) {
|
||||
parent = new EntityReference();
|
||||
parent.setId(UUID.fromString((String) parentMap.get("id")));
|
||||
parent.setId(parseUuid((String) parentMap.get("id")));
|
||||
parent.setType((String) parentMap.get("type"));
|
||||
parent.setName((String) parentMap.get("name"));
|
||||
parent.setFullyQualifiedName((String) parentMap.get("fullyQualifiedName"));
|
||||
|
|
@ -46,8 +46,9 @@ public final class SearchUtils {
|
|||
|
||||
PageHierarchy page = new PageHierarchy();
|
||||
|
||||
if (idStr != null) {
|
||||
page.withId(UUID.fromString(idStr));
|
||||
UUID pageId = parseUuid(idStr);
|
||||
if (pageId != null) {
|
||||
page.withId(pageId);
|
||||
}
|
||||
|
||||
if (pageTypeStr != null) {
|
||||
|
|
@ -61,4 +62,20 @@ public final class SearchUtils {
|
|||
|
||||
return page;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a UUID string safely — returns null for missing or malformed values so a single
|
||||
* bad hit does not break the entire hierarchy response.
|
||||
*/
|
||||
private static UUID parseUuid(String value) {
|
||||
if (value == null || value.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return UUID.fromString(value);
|
||||
} catch (IllegalArgumentException e) {
|
||||
LOG.warn("Ignoring malformed UUID in search hit: {}", value);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue