From 08e52b96c8ba2b923971e153c6bc4cc8aeca34df Mon Sep 17 00:00:00 2001 From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:59:26 +0530 Subject: [PATCH] Fix payload size issue (#27388) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix Payload Size issue, increase bufer * Handle single entitiy with > 10 mb * Single entity push * Normalize SQl Queries * Update generated TypeScript types * Add Tests * Fix Failing Test * Revert fixes * Fix Tests * Strip Lineage * Strip lIneage and make default 9 mb * Add Warn log on Large entity size * Review Comments * Remove hierarchical fields * remove team containing users * revert unwanted changes * Fix test failures from payload size default change - Update mock expectations in SearchIndexExecutorControlFlowTest and DistributedJobParticipantTest to use DEFAULT_BULK_PAYLOAD_SIZE_BYTES instead of hardcoded 104857600L (old 100MB default) - Remove "charts" from DashboardIndex excluded fields — charts are needed for search filters and column lineage resolution Co-Authored-By: Claude Opus 4.6 (1M context) * Fix hardcoded payload size fallback in DistributedJobParticipant Replace hardcoded 104857600L with SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES to use the centralized 9MB default consistently. Co-Authored-By: Claude Opus 4.6 (1M context) * Spotless * Update payLoadSize schema defaults to 9MB The JSON schema default for payLoadSize was 104857600 (100MB), which meant EventPublisherJob.getPayLoadSize() always returned 100MB instead of null, bypassing the DEFAULT_BULK_PAYLOAD_SIZE_BYTES fallback in DistributedJobParticipant. Align schema defaults with the 9MB bulk payload limit. Co-Authored-By: Claude Opus 4.6 (1M context) * Address review comments: fix log message, reuse getLineageData, fix test key algorithm - Fix misleading log in stripLineageForSize: report post-strip size, not imply bytes removed - Reuse getLineageData() in populateLineageData() instead of duplicating the DAO call and edge construction loop - Fix AddUpdateLineageScriptTest key algorithm to use maxKey+1 matching the Painless script, avoiding key collisions after deletions Co-Authored-By: Claude Opus 4.6 (1M context) * Address review: equals() in Painless, prune stale SQL keys, flush oversized single ops - Use .equals() instead of == for string comparison in REMOVE_LINEAGE_SCRIPT Painless to be explicit about value equality and null-safe (params on left) - Prune orphaned sqlQueryKey in ADD_UPDATE_LINEAGE when updating an edge with a different SQL query, preventing unbounded lineageSqlQueries growth - Restore currentBufferSize >= maxPayloadSizeBytes check in CustomBulkProcessor.add() so a single oversized operation is flushed immediately rather than sitting in the buffer - Update AddUpdateLineageScriptTest to mirror the new pruning logic Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: github-actions[bot] Co-authored-by: Claude Sonnet 4.6 --- .../searchIndex/ElasticSearchBulkSink.java | 97 ++++++- .../searchIndex/OpenSearchBulkSink.java | 92 ++++++- .../searchIndex/ReindexingConfiguration.java | 3 +- .../DistributedJobParticipant.java | 3 +- .../service/search/SearchClient.java | 78 +++++- .../service/search/SearchClusterMetrics.java | 10 +- .../search/SearchIndexRetryWorker.java | 4 +- .../service/search/SearchIndexUtils.java | 115 ++++++++ .../service/search/SearchRepository.java | 7 +- .../search/indexes/APICollectionIndex.java | 6 + .../search/indexes/ContainerIndex.java | 6 + .../search/indexes/DashboardIndex.java | 6 + .../service/search/indexes/DatabaseIndex.java | 7 + .../search/indexes/DatabaseSchemaIndex.java | 6 + .../search/indexes/GlossaryTermIndex.java | 6 + .../search/indexes/LlmServiceIndex.java | 6 + .../service/search/indexes/SearchIndex.java | 17 ++ .../service/search/indexes/TeamIndex.java | 2 +- .../SearchIndexExecutorControlFlowTest.java | 8 +- .../DistributedJobParticipantTest.java | 8 +- .../indexes/AddUpdateLineageScriptTest.java | 250 ++++++++++++++++++ .../search/indexes/SearchIndexTest.java | 151 +++++++++++ .../en/ai_agent_index_mapping.json | 4 + .../ai_governance_policy_index_mapping.json | 4 + .../en/api_collection_index_mapping.json | 7 + .../en/api_endpoint_index_mapping.json | 7 + .../elasticsearch/en/chart_index_mapping.json | 7 + .../en/container_index_mapping.json | 7 + .../dashboard_data_model_index_mapping.json | 7 + .../en/dashboard_index_mapping.json | 7 + .../en/directory_index_mapping.json | 7 + .../elasticsearch/en/file_index_mapping.json | 7 + .../en/llm_model_index_mapping.json | 4 + .../en/metric_index_mapping.json | 7 + .../en/mlmodel_index_mapping.json | 7 + .../en/pipeline_index_mapping.json | 7 + .../en/prompt_template_index_mapping.json | 4 + .../en/search_entity_index_mapping.json | 7 + .../en/spreadsheet_index_mapping.json | 7 + .../en/stored_procedure_index_mapping.json | 7 + .../elasticsearch/en/table_index_mapping.json | 7 + .../elasticsearch/en/team_index_mapping.json | 72 +---- .../elasticsearch/en/topic_index_mapping.json | 7 + .../en/worksheet_index_mapping.json | 7 + .../jp/ai_agent_index_mapping.json | 4 + .../ai_governance_policy_index_mapping.json | 4 + .../jp/api_collection_index_mapping.json | 7 + .../jp/api_endpoint_index_mapping.json | 7 + .../elasticsearch/jp/chart_index_mapping.json | 7 + .../jp/container_index_mapping.json | 7 + .../dashboard_data_model_index_mapping.json | 7 + .../jp/dashboard_index_mapping.json | 7 + .../jp/directory_index_mapping.json | 7 + .../elasticsearch/jp/file_index_mapping.json | 7 + .../jp/llm_model_index_mapping.json | 4 + .../jp/metric_index_mapping.json | 7 + .../jp/mlmodel_index_mapping.json | 7 + .../jp/pipeline_index_mapping.json | 7 + .../jp/prompt_template_index_mapping.json | 4 + .../jp/search_entity_index_mapping.json | 7 + .../jp/spreadsheet_index_mapping.json | 7 + .../jp/stored_procedure_index_mapping.json | 7 + .../elasticsearch/jp/table_index_mapping.json | 7 + .../elasticsearch/jp/team_index_mapping.json | 72 +---- .../elasticsearch/jp/topic_index_mapping.json | 7 + .../jp/worksheet_index_mapping.json | 7 + .../ru/ai_agent_index_mapping.json | 4 + .../ai_governance_policy_index_mapping.json | 4 + .../ru/api_collection_index_mapping.json | 7 + .../ru/api_endpoint_index_mapping.json | 7 + .../elasticsearch/ru/chart_index_mapping.json | 7 + .../ru/container_index_mapping.json | 7 + .../dashboard_data_model_index_mapping.json | 7 + .../ru/dashboard_index_mapping.json | 7 + .../ru/directory_index_mapping.json | 7 + .../elasticsearch/ru/file_index_mapping.json | 7 + .../ru/llm_model_index_mapping.json | 4 + .../ru/metric_index_mapping.json | 7 + .../ru/mlmodel_index_mapping.json | 7 + .../ru/pipeline_index_mapping.json | 7 + .../ru/prompt_template_index_mapping.json | 4 + .../ru/search_entity_index_mapping.json | 7 + .../ru/spreadsheet_index_mapping.json | 7 + .../ru/stored_procedure_index_mapping.json | 7 + .../elasticsearch/ru/table_index_mapping.json | 7 + .../elasticsearch/ru/team_index_mapping.json | 72 +---- .../elasticsearch/ru/topic_index_mapping.json | 7 + .../ru/worksheet_index_mapping.json | 7 + .../zh/ai_agent_index_mapping.json | 4 + .../ai_governance_policy_index_mapping.json | 4 + .../zh/api_collection_index_mapping.json | 7 + .../zh/api_endpoint_index_mapping.json | 7 + .../elasticsearch/zh/chart_index_mapping.json | 7 + .../zh/container_index_mapping.json | 7 + .../dashboard_data_model_index_mapping.json | 7 + .../zh/dashboard_index_mapping.json | 7 + .../zh/directory_index_mapping.json | 7 + .../elasticsearch/zh/file_index_mapping.json | 7 + .../zh/llm_model_index_mapping.json | 4 + .../zh/metric_index_mapping.json | 7 + .../zh/mlmodel_index_mapping.json | 7 + .../zh/pipeline_index_mapping.json | 7 + .../zh/prompt_template_index_mapping.json | 4 + .../zh/search_entity_index_mapping.json | 7 + .../zh/spreadsheet_index_mapping.json | 7 + .../zh/stored_procedure_index_mapping.json | 7 + .../elasticsearch/zh/table_index_mapping.json | 7 + .../elasticsearch/zh/team_index_mapping.json | 72 +---- .../elasticsearch/zh/topic_index_mapping.json | 7 + .../zh/worksheet_index_mapping.json | 7 + .../schema/api/lineage/esLineageData.json | 4 + .../internal/searchIndexingAppConfig.json | 2 +- .../json/schema/system/eventPublisherJob.json | 2 +- .../generated/api/lineage/esLineageData.ts | 6 + 114 files changed, 1415 insertions(+), 315 deletions(-) create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/AddUpdateLineageScriptTest.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchBulkSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchBulkSink.java index 24ad18dd536..f1e7c990b69 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchBulkSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ElasticSearchBulkSink.java @@ -46,6 +46,7 @@ import org.openmetadata.service.apps.bundles.searchIndex.stats.StatsResult; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.exception.SearchIndexException; import org.openmetadata.service.search.ReindexContext; +import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.elasticsearch.ElasticSearchClient; import org.openmetadata.service.search.elasticsearch.EsUtils; @@ -95,6 +96,7 @@ public class ElasticSearchBulkSink implements BulkSink { private final ElasticSearchClient searchClient; protected final SearchRepository searchRepository; + private final long maxPayloadSizeBytes; private final CustomBulkProcessor bulkProcessor; private final StepStats stats = new StepStats(); @@ -133,6 +135,7 @@ public class ElasticSearchBulkSink implements BulkSink { this.searchClient = (ElasticSearchClient) searchRepository.getSearchClient(); this.batchSize = batchSize; this.maxConcurrentRequests = maxConcurrentRequests; + this.maxPayloadSizeBytes = maxPayloadSizeBytes; // Initialize stats stats.withTotalRecords(0).withSuccessRecords(0).withFailedRecords(0); @@ -294,7 +297,7 @@ public class ElasticSearchBulkSink implements BulkSink { return null; } - private static final int BULK_OPERATION_METADATA_OVERHEAD = 50; + private static final int BULK_OPERATION_METADATA_OVERHEAD = 150; private void addEntity( EntityInterface entity, String indexName, boolean recreateIndex, StageStatsTracker tracker) { @@ -303,16 +306,53 @@ public class ElasticSearchBulkSink implements BulkSink { Object searchIndexDoc = Entity.buildSearchIndex(entityType, entity).buildSearchIndexDoc(); String json = JsonUtils.pojoToJson(searchIndexDoc); String docId = entity.getId().toString(); - long estimatedSize = - (long) json.getBytes(StandardCharsets.UTF_8).length + BULK_OPERATION_METADATA_OVERHEAD; + long rawDocSize = (long) json.getBytes(StandardCharsets.UTF_8).length; + long estimatedSize = rawDocSize + BULK_OPERATION_METADATA_OVERHEAD; + if (rawDocSize > 1024 * 1024) { + LOG.warn( + "Large indexed doc: entityType={}, docId={}, size={}MB", + entityType, + docId, + rawDocSize / (1024 * 1024)); + } + + if (estimatedSize > maxPayloadSizeBytes) { + long sizeLimit = maxPayloadSizeBytes - BULK_OPERATION_METADATA_OVERHEAD; + json = SearchIndexUtils.stripLineageForSize(json, sizeLimit, docId, entityType); + rawDocSize = json.getBytes(StandardCharsets.UTF_8).length; + estimatedSize = rawDocSize + BULK_OPERATION_METADATA_OVERHEAD; + } + + if (estimatedSize > maxPayloadSizeBytes) { + LOG.warn( + "Document {} of type {} is too large for bulk ({} bytes), sending directly", + docId, + entityType, + rawDocSize); + totalSubmitted.incrementAndGet(); + if (tracker != null) { + tracker.incrementPendingSink(); + } + indexDocumentDirectly(indexName, docId, json, entityType, tracker); + processSuccess.incrementAndGet(); + if (tracker != null) { + tracker.recordProcess(StatsResult.SUCCESS); + } + return; + } + + final String indexableJson = json; BulkOperation operation; if (recreateIndex) { operation = BulkOperation.of( op -> op.index( - idx -> idx.index(indexName).id(docId).document(EsUtils.toJsonData(json)))); + idx -> + idx.index(indexName) + .id(docId) + .document(EsUtils.toJsonData(indexableJson)))); } else { operation = BulkOperation.of( @@ -321,7 +361,10 @@ public class ElasticSearchBulkSink implements BulkSink { upd -> upd.index(indexName) .id(docId) - .action(a -> a.doc(EsUtils.toJsonData(json)).docAsUpsert(true)))); + .action( + a -> + a.doc(EsUtils.toJsonData(indexableJson)) + .docAsUpsert(true)))); } if (tracker != null) { tracker.incrementPendingSink(); @@ -369,6 +412,42 @@ public class ElasticSearchBulkSink implements BulkSink { } } + private void indexDocumentDirectly( + String indexName, String docId, String json, String entityType, StageStatsTracker tracker) { + try { + searchClient + .getNewClient() + .index(idx -> idx.index(indexName).id(docId).document(EsUtils.toJsonData(json))); + totalSuccess.incrementAndGet(); + updateStats(); + if (tracker != null) { + tracker.recordSink(StatsResult.SUCCESS); + } + } catch (Exception e) { + LOG.error( + "Direct index failed for document {} of type {}: {}", + docId, + entityType, + e.getMessage(), + e); + totalFailed.incrementAndGet(); + updateStats(); + if (tracker != null) { + tracker.recordSink(StatsResult.FAILED); + } + if (failureCallback != null) { + failureCallback.onFailure( + entityType, + docId, + null, + String.format( + "Document too large for bulk (%d bytes); direct index failed: %s", + json.getBytes(StandardCharsets.UTF_8).length, e.getMessage()), + IndexingFailureRecorder.FailureStage.SINK); + } + } + } + private void addTimeSeriesEntity( EntityTimeSeriesInterface entity, String indexName, @@ -755,6 +834,8 @@ public class ElasticSearchBulkSink implements BulkSink { throw new IllegalStateException("Bulk processor is closed"); } + totalSubmitted.incrementAndGet(); + if (docId != null) { if (entityType != null) { docIdToEntityType.put(docId, entityType); @@ -766,6 +847,10 @@ public class ElasticSearchBulkSink implements BulkSink { long operationSize = estimatedSizeBytes > 0 ? estimatedSizeBytes : estimateOperationSize(operation); + + if (!buffer.isEmpty() && currentBufferSize + operationSize >= maxPayloadSizeBytes) { + flushInternal(); + } buffer.add(operation); currentBufferSize += operationSize; @@ -852,8 +937,6 @@ public class ElasticSearchBulkSink implements BulkSink { long executionId = executionIdCounter.incrementAndGet(); int numberOfActions = toFlush.size(); - totalSubmitted.addAndGet(numberOfActions); - LOG.debug("Executing bulk request {} with {} actions", executionId, numberOfActions); try { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchBulkSink.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchBulkSink.java index b23a6a7b915..55d24d0485d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchBulkSink.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/OpenSearchBulkSink.java @@ -42,6 +42,7 @@ import org.openmetadata.service.apps.bundles.searchIndex.stats.StatsResult; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.exception.SearchIndexException; import org.openmetadata.service.search.ReindexContext; +import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.indexes.ColumnSearchIndex; import org.openmetadata.service.search.opensearch.OpenSearchClient; @@ -107,6 +108,7 @@ public class OpenSearchBulkSink implements BulkSink { private final OpenSearchClient searchClient; protected final SearchRepository searchRepository; + private final long maxPayloadSizeBytes; private final CustomBulkProcessor bulkProcessor; private final StepStats stats = new StepStats(); @@ -152,6 +154,7 @@ public class OpenSearchBulkSink implements BulkSink { this.searchClient = (OpenSearchClient) searchRepository.getSearchClient(); this.batchSize = batchSize; this.maxConcurrentRequests = maxConcurrentRequests; + this.maxPayloadSizeBytes = maxPayloadSizeBytes; // Initialize stats stats.withTotalRecords(0).withSuccessRecords(0).withFailedRecords(0); @@ -333,7 +336,7 @@ public class OpenSearchBulkSink implements BulkSink { return null; } - private static final int BULK_OPERATION_METADATA_OVERHEAD = 50; + private static final int BULK_OPERATION_METADATA_OVERHEAD = 150; private void addEntity( EntityInterface entity, @@ -354,10 +357,43 @@ public class OpenSearchBulkSink implements BulkSink { String finalJson = json; String docId = entity.getId().toString(); - long estimatedSize = - (long) finalJson.getBytes(StandardCharsets.UTF_8).length - + BULK_OPERATION_METADATA_OVERHEAD; + long rawDocSize = (long) finalJson.getBytes(StandardCharsets.UTF_8).length; + long estimatedSize = rawDocSize + BULK_OPERATION_METADATA_OVERHEAD; + if (rawDocSize > 1024 * 1024) { + LOG.warn( + "Large indexed doc: entityType={}, docId={}, size={}MB", + entityType, + docId, + rawDocSize / (1024 * 1024)); + } + + if (estimatedSize > maxPayloadSizeBytes) { + long sizeLimit = maxPayloadSizeBytes - BULK_OPERATION_METADATA_OVERHEAD; + finalJson = SearchIndexUtils.stripLineageForSize(finalJson, sizeLimit, docId, entityType); + rawDocSize = finalJson.getBytes(StandardCharsets.UTF_8).length; + estimatedSize = rawDocSize + BULK_OPERATION_METADATA_OVERHEAD; + } + + if (estimatedSize > maxPayloadSizeBytes) { + LOG.warn( + "Document {} of type {} is too large for bulk ({} bytes), sending directly", + docId, + entityType, + rawDocSize); + totalSubmitted.incrementAndGet(); + if (tracker != null) { + tracker.incrementPendingSink(); + } + indexDocumentDirectly(indexName, docId, finalJson, entityType, tracker); + processSuccess.incrementAndGet(); + if (tracker != null) { + tracker.recordProcess(StatsResult.SUCCESS); + } + return; + } + + final String indexableJson = finalJson; BulkOperation operation; if (recreateIndex) { operation = @@ -367,7 +403,7 @@ public class OpenSearchBulkSink implements BulkSink { idx -> idx.index(indexName) .id(docId) - .document(OsUtils.toJsonData(finalJson)))); + .document(OsUtils.toJsonData(indexableJson)))); } else { operation = BulkOperation.of( @@ -376,7 +412,7 @@ public class OpenSearchBulkSink implements BulkSink { upd -> upd.index(indexName) .id(docId) - .document(OsUtils.toJsonData(finalJson)) + .document(OsUtils.toJsonData(indexableJson)) .docAsUpsert(true))); } if (tracker != null) { @@ -425,6 +461,42 @@ public class OpenSearchBulkSink implements BulkSink { } } + private void indexDocumentDirectly( + String indexName, String docId, String json, String entityType, StageStatsTracker tracker) { + try { + searchClient + .getNewClient() + .index(idx -> idx.index(indexName).id(docId).document(OsUtils.toJsonData(json))); + totalSuccess.incrementAndGet(); + updateStats(); + if (tracker != null) { + tracker.recordSink(StatsResult.SUCCESS); + } + } catch (Exception e) { + LOG.error( + "Direct index failed for document {} of type {}: {}", + docId, + entityType, + e.getMessage(), + e); + totalFailed.incrementAndGet(); + updateStats(); + if (tracker != null) { + tracker.recordSink(StatsResult.FAILED); + } + if (failureCallback != null) { + failureCallback.onFailure( + entityType, + docId, + null, + String.format( + "Document too large for bulk (%d bytes); direct index failed: %s", + json.getBytes(StandardCharsets.UTF_8).length, e.getMessage()), + IndexingFailureRecorder.FailureStage.SINK); + } + } + } + private void addTimeSeriesEntity( EntityTimeSeriesInterface entity, String indexName, @@ -899,6 +971,8 @@ public class OpenSearchBulkSink implements BulkSink { throw new IllegalStateException("Bulk processor is closed"); } + totalSubmitted.incrementAndGet(); + if (docId != null) { if (entityType != null) { docIdToEntityType.put(docId, entityType); @@ -910,6 +984,10 @@ public class OpenSearchBulkSink implements BulkSink { long operationSize = estimatedSizeBytes > 0 ? estimatedSizeBytes : estimateOperationSize(operation); + + if (!buffer.isEmpty() && currentBufferSize + operationSize >= maxPayloadSizeBytes) { + flushInternal(); + } buffer.add(operation); currentBufferSize += operationSize; @@ -1001,8 +1079,6 @@ public class OpenSearchBulkSink implements BulkSink { long executionId = executionIdCounter.incrementAndGet(); int numberOfActions = toFlush.size(); - totalSubmitted.addAndGet(numberOfActions); - LOG.debug("Executing bulk request {} with {} actions", executionId, numberOfActions); try { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ReindexingConfiguration.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ReindexingConfiguration.java index 10e5842f4e1..2426e63c367 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ReindexingConfiguration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ReindexingConfiguration.java @@ -46,7 +46,8 @@ public record ReindexingConfiguration( private static final int DEFAULT_PRODUCER_THREADS = 1; private static final int DEFAULT_QUEUE_SIZE = 100; private static final int DEFAULT_MAX_CONCURRENT_REQUESTS = 100; - private static final long DEFAULT_PAYLOAD_SIZE = 104857600L; + private static final long DEFAULT_PAYLOAD_SIZE = + SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES; private static final int DEFAULT_FIELD_FETCH_THREADS = 0; private static final int DEFAULT_DOC_BUILD_THREADS = 0; private static final long DEFAULT_STATS_INTERVAL_MS = 0; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedJobParticipant.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedJobParticipant.java index 941e5981c15..c0dc8ce20b4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedJobParticipant.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedJobParticipant.java @@ -28,6 +28,7 @@ import org.openmetadata.service.apps.bundles.searchIndex.IndexingFailureRecorder import org.openmetadata.service.cache.CacheConfig; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.search.SearchClusterMetrics; import org.openmetadata.service.search.SearchRepository; /** @@ -333,7 +334,7 @@ public class DistributedJobParticipant implements Managed { : 100, job.getJobConfiguration().getPayLoadSize() != null ? job.getJobConfiguration().getPayLoadSize() - : 104857600L); + : SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES); int batchSize = job.getJobConfiguration().getBatchSize() != null diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index f8927b6cb7a..9a052b19323 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -237,23 +237,95 @@ public interface SearchClient """; String REMOVE_LINEAGE_SCRIPT = - "ctx._source.upstreamLineage.removeIf(lineage -> lineage.docUniqueId == params.docUniqueId)"; + """ + def removedKeys = new HashSet(); + for (def lineage : ctx._source.upstreamLineage) { + if (params.docUniqueId.equals(lineage.docUniqueId) && lineage.containsKey('sqlQueryKey')) { + removedKeys.add(lineage.sqlQueryKey); + } + } + ctx._source.upstreamLineage.removeIf(lineage -> params.docUniqueId.equals(lineage.docUniqueId)); + if (!removedKeys.isEmpty() && ctx._source.containsKey('lineageSqlQueries') && ctx._source.lineageSqlQueries != null) { + def sqlMap = ctx._source.lineageSqlQueries; + def usedKeys = new HashSet(); + for (def lineage : ctx._source.upstreamLineage) { + if (lineage.containsKey('sqlQueryKey')) { + usedKeys.add(lineage.sqlQueryKey); + } + } + removedKeys.removeAll(usedKeys); + for (def key : removedKeys) { + sqlMap.remove(key); + } + } + """; String REMOVE_ENTITY_RELATIONSHIP = "ctx._source.upstreamEntityRelationship.removeIf(relationship -> relationship.docId == params.docId)"; String ADD_UPDATE_LINEAGE = """ + // Dedup sqlQuery into the doc-level lineageSqlQueries map. + // If the incoming edge carries a sqlQuery, store it once in lineageSqlQueries + // keyed by a sequential integer, then replace sqlQuery with sqlQueryKey on the edge. + def rawSql = params.lineageData['sqlQuery']; + Map edgeData; + if (rawSql != null && !rawSql.isEmpty()) { + if (!ctx._source.containsKey('lineageSqlQueries') || ctx._source['lineageSqlQueries'] == null) { + ctx._source['lineageSqlQueries'] = new HashMap(); + } + def sqlMap = ctx._source['lineageSqlQueries']; + def sqlKey = null; + for (def entry : sqlMap.entrySet()) { + if (entry.getValue().equals(rawSql)) { + sqlKey = entry.getKey(); + break; + } + } + if (sqlKey == null) { + def maxKey = 0; + for (def k : sqlMap.keySet()) { + def kInt = Integer.parseInt(k); + if (kInt > maxKey) maxKey = kInt; + } + sqlKey = String.valueOf(maxKey + 1); + sqlMap.put(sqlKey, rawSql); + } + edgeData = new HashMap(); + edgeData.putAll(params.lineageData); + edgeData.put('sqlQueryKey', sqlKey); + edgeData.remove('sqlQuery'); + } else { + edgeData = params.lineageData; + } + // Replace or add the edge, capturing the old sqlQueryKey for cleanup. + def oldSqlQueryKey = null; boolean docIdExists = false; for (int i = 0; i < ctx._source.upstreamLineage.size(); i++) { if (ctx._source.upstreamLineage[i].docUniqueId.equalsIgnoreCase(params.lineageData.docUniqueId)) { - ctx._source.upstreamLineage[i] = params.lineageData; + if (ctx._source.upstreamLineage[i].containsKey('sqlQueryKey')) { + oldSqlQueryKey = ctx._source.upstreamLineage[i].sqlQueryKey; + } + ctx._source.upstreamLineage[i] = edgeData; docIdExists = true; break; } } if (!docIdExists) { - ctx._source.upstreamLineage.add(params.lineageData); + ctx._source.upstreamLineage.add(edgeData); + } + // Prune the old SQL key if it changed and is no longer used by any edge. + if (oldSqlQueryKey != null && !oldSqlQueryKey.equals(edgeData.containsKey('sqlQueryKey') ? edgeData.get('sqlQueryKey') : null)) { + boolean stillUsed = false; + for (def lineage : ctx._source.upstreamLineage) { + if (lineage.containsKey('sqlQueryKey') && oldSqlQueryKey.equals(lineage.sqlQueryKey)) { + stillUsed = true; + break; + } + } + if (!stillUsed && ctx._source.containsKey('lineageSqlQueries') && ctx._source.lineageSqlQueries != null) { + ctx._source.lineageSqlQueries.remove(oldSqlQueryKey); + } } """; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClusterMetrics.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClusterMetrics.java index 6d12cb9afe5..eca4c68e4a0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClusterMetrics.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClusterMetrics.java @@ -34,7 +34,9 @@ public class SearchClusterMetrics { public static final long DEFAULT_HEAP_USED_BYTES = 512L * 1024 * 1024; // 512 MB public static final long DEFAULT_HEAP_MAX_BYTES = 1024L * 1024 * 1024; // 1 GB public static final long DEFAULT_MAX_CONTENT_LENGTH = - 10 * 1024 * 1024L; // Conservative 10MB default + 10 * 1024 * 1024L; // Conservative 10MB default (AWS OpenSearch hard limit) + // Safe bulk payload threshold: 90% of max_content_length to leave headroom for HTTP framing + public static final long DEFAULT_BULK_PAYLOAD_SIZE_BYTES = DEFAULT_MAX_CONTENT_LENGTH * 9 / 10; public static SearchClusterMetrics fetchClusterMetrics( SearchRepository searchRepository, long totalEntities, int maxDbConnections) { @@ -445,9 +447,9 @@ public class SearchClusterMetrics { long usedHeap = totalHeap - freeHeap; double heapUsagePercent = (maxHeap > 0) ? (double) usedHeap / maxHeap * 100 : 50.0; - // Default to conservative 10MB for AWS-managed clusters if we can't fetch from cluster - long maxContentLength = DEFAULT_MAX_CONTENT_LENGTH; // Conservative 10MB default - long maxPayloadSize = DEFAULT_MAX_CONTENT_LENGTH; // Conservative 10MB default + // AWS-managed clusters expose max_content_length=10MB; use 90% as bulk threshold for headroom + long maxContentLength = DEFAULT_MAX_CONTENT_LENGTH; + long maxPayloadSize = DEFAULT_BULK_PAYLOAD_SIZE_BYTES; try { if (searchRepository != null) { SearchClient searchClient = searchRepository.getSearchClient(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexRetryWorker.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexRetryWorker.java index 43dfa368efa..1c4348d0c81 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexRetryWorker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexRetryWorker.java @@ -481,7 +481,9 @@ public class SearchIndexRetryWorker implements Managed { Set failedEntityIds = ConcurrentHashMap.newKeySet(); AtomicReference firstFailureDetail = new AtomicReference<>(); - BulkSink bulkSink = searchRepository.createBulkSink(200, 5, 10L * 1024L * 1024L); + BulkSink bulkSink = + searchRepository.createBulkSink( + 200, 5, SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES); bulkSink.setFailureCallback( (entityType, entityId, entityFqn, errorMessage, stage) -> { if (entityId != null && !entityId.isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java index cf7c4d0837c..e1b2fb35887 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java @@ -3,15 +3,18 @@ package org.openmetadata.service.search; import static org.openmetadata.service.search.SearchUtils.getAggregationBuckets; import static org.openmetadata.service.search.SearchUtils.getAggregationObject; +import com.fasterxml.jackson.core.type.TypeReference; import jakarta.json.JsonArray; import jakarta.json.JsonNumber; import jakarta.json.JsonObject; import jakarta.json.JsonString; import jakarta.json.JsonValue; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -21,6 +24,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.ColumnsEntityInterface; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.lineage.EsLineageData; import org.openmetadata.schema.tests.DataQualityReport; import org.openmetadata.schema.tests.Datum; import org.openmetadata.schema.tests.type.DataQualityReportMetadata; @@ -44,6 +48,117 @@ public final class SearchIndexUtils { private SearchIndexUtils() {} + /** + * Deduplicates identical SQL queries across lineage edges in-place. + * + *

Each unique SQL text is assigned a sequential integer key ("1", "2", …). Every edge that + * carries that SQL has its {@code sqlQuery} cleared and {@code sqlQueryKey} set to the shared + * key. The returned map contains {@code key → sqlText} for all unique SQLs found. + * + *

Edges with no SQL are left untouched. + */ + public static Map deduplicateSqlAcrossEdges(List edges) { + Map sqlTextToKey = new LinkedHashMap<>(); + Map sqlQueries = new LinkedHashMap<>(); + int[] counter = {0}; + + for (EsLineageData edge : edges) { + String sql = edge.getSqlQuery(); + if (sql != null && !sql.isEmpty()) { + String key = + sqlTextToKey.computeIfAbsent( + sql, + k -> { + String newKey = String.valueOf(++counter[0]); + sqlQueries.put(newKey, sql); + return newKey; + }); + edge.setSqlQueryKey(key); + edge.setSqlQuery(null); + } + } + + return sqlQueries; + } + + /** + * Progressively strips lineage fields from a search document JSON to bring it under maxBytes. + * + *

Stripping order: lineageSqlQueries first (retains topology), then upstreamLineage. + * Returns the (possibly stripped) JSON — caller must re-check size and handle the still-oversized + * case. + */ + public static String stripLineageForSize( + String json, long maxBytes, String docId, String entityType) { + if (json.getBytes(StandardCharsets.UTF_8).length <= maxBytes) { + return json; + } + TypeReference> mapType = new TypeReference<>() {}; + Map doc = JsonUtils.readValue(json, mapType); + if (doc.remove("lineageSqlQueries") != null) { + stripSqlQueryKeysFromEdges(doc); + json = JsonUtils.pojoToJson(doc); + int sizeAfterStrip = json.getBytes(StandardCharsets.UTF_8).length; + LOG.warn( + "Document {} ({}) too large, stripped lineageSqlQueries (size now {} bytes)", + docId, + entityType, + sizeAfterStrip); + if (sizeAfterStrip <= maxBytes) { + return json; + } + } + doc.remove("upstreamLineage"); + json = JsonUtils.pojoToJson(doc); + LOG.warn( + "Document {} ({}) still too large, stripped upstreamLineage (size now {} bytes)", + docId, + entityType, + json.getBytes(StandardCharsets.UTF_8).length); + return json; + } + + public static Map stripDocMapIfOversized( + Map doc, long maxBytes, String docId, String entityType) { + String json = JsonUtils.pojoToJson(doc); + if (json.getBytes(StandardCharsets.UTF_8).length <= maxBytes) { + return doc; + } + if (doc.remove("lineageSqlQueries") != null) { + stripSqlQueryKeysFromEdges(doc); + json = JsonUtils.pojoToJson(doc); + int strippedSize = json.getBytes(StandardCharsets.UTF_8).length; + LOG.warn( + "Live index doc {} ({}) too large, stripped lineageSqlQueries ({} bytes)", + docId, + entityType, + strippedSize); + if (strippedSize <= maxBytes) { + return doc; + } + } + if (doc.remove("upstreamLineage") != null) { + LOG.warn( + "Live index doc {} ({}) still too large, stripped upstreamLineage ({} bytes)", + docId, + entityType, + JsonUtils.pojoToJson(doc).getBytes(StandardCharsets.UTF_8).length); + } + return doc; + } + + @SuppressWarnings("unchecked") + private static void stripSqlQueryKeysFromEdges(Map doc) { + Object lineage = doc.get("upstreamLineage"); + if (lineage instanceof List edges) { + for (Object edge : edges) { + if (edge instanceof Map edgeMap) { + ((Map) edgeMap).remove("sqlQueryKey"); + } + } + } + } + public static List parseFollowers(List followersRef) { if (followersRef == null) { return Collections.emptyList(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index b480a07eaaa..9028199b2b3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -1081,10 +1081,11 @@ public class SearchRepository { } SearchIndex elasticSearchIndex = searchIndexFactory.buildIndex(entityType, entity); doc = elasticSearchIndex.buildSearchIndexDoc(); + doc = + SearchIndexUtils.stripDocMapIfOversized( + doc, SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES, entityId, entityType); } - // Use synchronous update to ensure tests pass - // TODO: Consider using async updates with proper wait mechanisms in tests searchClient.updateEntity(indexMapping.getIndexName(clusterAlias), entityId, doc, scriptTxt); if (Entity.TABLE.equals(entityType)) { @@ -1242,7 +1243,7 @@ public class SearchRepository { int batchSize = 100; int maxConcurrentRequests = 5; - long maxPayloadSizeBytes = 10 * 1024 * 1024; // 10MB + long maxPayloadSizeBytes = SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES; // Process each entity type separately to ensure correct index routing for (Map.Entry> entry : entitiesByType.entrySet()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APICollectionIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APICollectionIndex.java index 4896235a038..7fc2bf181bc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APICollectionIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APICollectionIndex.java @@ -1,6 +1,7 @@ package org.openmetadata.service.search.indexes; import java.util.Map; +import java.util.Set; import org.openmetadata.schema.entity.data.APICollection; import org.openmetadata.service.Entity; @@ -16,6 +17,11 @@ public record APICollectionIndex(APICollection apiCollection) implements Taggabl return Entity.API_COLLECTION; } + @Override + public Set getExcludedFields() { + return Set.of("apiEndpoints"); + } + public Map buildSearchIndexDocInternal(Map doc) { return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java index 6f655b4968c..5d6f7aa2f6c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java @@ -14,6 +14,7 @@ import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.FlattenColumn; public record ContainerIndex(Container container) implements ColumnIndex, DataAssetIndex { + @Override public Object getEntity() { return container; @@ -29,6 +30,11 @@ public record ContainerIndex(Container container) implements ColumnIndex, DataAs return container.getServiceType(); } + @Override + public Set getExcludedFields() { + return Set.of("children"); + } + public Map buildSearchIndexDocInternal(Map doc) { if (container.getDataModel() != null && container.getDataModel().getColumns() != null) { List cols = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java index 58517984997..c9b0bdab7f0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java @@ -1,6 +1,7 @@ package org.openmetadata.service.search.indexes; import java.util.Map; +import java.util.Set; import org.openmetadata.schema.entity.data.Dashboard; import org.openmetadata.service.Entity; @@ -26,6 +27,11 @@ public class DashboardIndex implements DataAssetIndex { return dashboard.getServiceType(); } + @Override + public Set getExcludedFields() { + return Set.of("dataModels"); + } + public Map buildSearchIndexDocInternal(Map doc) { return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java index 935da07f0da..655dd00a51b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java @@ -1,10 +1,12 @@ package org.openmetadata.service.search.indexes; import java.util.Map; +import java.util.Set; import org.openmetadata.schema.entity.data.Database; import org.openmetadata.service.Entity; public record DatabaseIndex(Database database) implements TaggableIndex { + @Override public Object getEntity() { return database; @@ -15,6 +17,11 @@ public record DatabaseIndex(Database database) implements TaggableIndex { return Entity.DATABASE; } + @Override + public Set getExcludedFields() { + return Set.of("databaseSchemas"); + } + public Map buildSearchIndexDocInternal(Map doc) { return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java index 9c8ac6e6eed..6f094d90f99 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java @@ -1,6 +1,7 @@ package org.openmetadata.service.search.indexes; import java.util.Map; +import java.util.Set; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.service.Entity; @@ -16,6 +17,11 @@ public record DatabaseSchemaIndex(DatabaseSchema databaseSchema) implements Tagg return Entity.DATABASE_SCHEMA; } + @Override + public Set getExcludedFields() { + return Set.of("tables"); + } + public Map buildSearchIndexDocInternal(Map doc) { return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java index 9a497749a2a..50ca62faac1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java @@ -1,6 +1,7 @@ package org.openmetadata.service.search.indexes; import java.util.Map; +import java.util.Set; import org.openmetadata.schema.entity.data.Glossary; import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.type.Include; @@ -23,6 +24,11 @@ public class GlossaryTermIndex implements TaggableIndex { return Entity.GLOSSARY_TERM; } + @Override + public Set getExcludedFields() { + return Set.of("children"); + } + public Map buildSearchIndexDocInternal(Map doc) { if (doc.containsKey("glossary") && glossaryTerm.getGlossary() != null) { @SuppressWarnings("unchecked") diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/LlmServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/LlmServiceIndex.java index 82b141127fd..0d423ea311a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/LlmServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/LlmServiceIndex.java @@ -1,6 +1,7 @@ package org.openmetadata.service.search.indexes; import java.util.Map; +import java.util.Set; import org.openmetadata.schema.entity.services.LLMService; import org.openmetadata.service.Entity; @@ -16,6 +17,11 @@ public record LlmServiceIndex(LLMService llmService) implements TaggableIndex, L return Entity.LLM_SERVICE; } + @Override + public Set getExcludedFields() { + return Set.of("models"); + } + public Map buildSearchIndexDocInternal(Map doc) { return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index ad6235a63a6..e78df543638 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -254,6 +254,23 @@ public interface SearchIndex { return data; } + /** + * Populates upstreamLineage and lineageSqlQueries in the given search doc map. + * + *

Identical SQL queries across edges are deduplicated: the full text is stored once in + * lineageSqlQueries keyed by a sequential integer, and each edge carries only the key via + * sqlQueryKey. Edges with unique SQL still get their SQL stored (and keyed). The authoritative + * per-edge SQL remains in the database; this deduplication is search-doc-local. + */ + static void populateLineageData(Map doc, EntityReference entity) { + List edges = getLineageData(entity); + Map sqlQueries = SearchIndexUtils.deduplicateSqlAcrossEdges(edges); + doc.put("upstreamLineage", edges); + if (!sqlQueries.isEmpty()) { + doc.put("lineageSqlQueries", sqlQueries); + } + } + static List> populateUpstreamEntityRelationshipData(Table entity) { List> upstreamRelationships = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java index 7583b6d7855..58fad179c7e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java @@ -7,7 +7,7 @@ import org.openmetadata.service.Entity; public class TeamIndex implements SearchIndex { final Team team; - final Set excludeFields = Set.of("owns"); + final Set excludeFields = Set.of("owns", "users", "defaultRoles", "inheritedRoles"); public TeamIndex(Team team) { this.team = team; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexExecutorControlFlowTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexExecutorControlFlowTest.java index df5c0d62c0c..b530e947aed 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexExecutorControlFlowTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexExecutorControlFlowTest.java @@ -64,6 +64,7 @@ import org.openmetadata.service.search.DefaultRecreateHandler; import org.openmetadata.service.search.EntityReindexContext; import org.openmetadata.service.search.RecreateIndexHandler; import org.openmetadata.service.search.ReindexContext; +import org.openmetadata.service.search.SearchClusterMetrics; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.RestUtil; @@ -893,7 +894,9 @@ class SearchIndexExecutorControlFlowTest { when(collectionDAO.searchIndexFailureDAO()).thenReturn(failureDao); when(entityRepository.getDao()).thenReturn(entityDao); when(entityDao.listCount(any(ListFilter.class))).thenReturn(0); - when(searchRepository.createBulkSink(100, 100, 104857600L)).thenReturn(sink); + when(searchRepository.createBulkSink( + 100, 100, SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES)) + .thenReturn(sink); when(searchRepository.createReindexHandler()).thenReturn(handler); when(handler.reCreateIndexes(Set.of(Entity.TABLE))).thenReturn(recreateContext); executor.addListener(listener); @@ -931,7 +934,8 @@ class SearchIndexExecutorControlFlowTest { when(jobContext.getJobId()).thenReturn(UUID.randomUUID()); when(entityRepository.getDao()).thenReturn(entityDao); when(entityDao.listCount(any(ListFilter.class))).thenReturn(0); - when(searchRepository.createBulkSink(100, 100, 104857600L)) + when(searchRepository.createBulkSink( + 100, 100, SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES)) .thenThrow(new IllegalStateException("sink init failed")); executor.addListener(listener); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedJobParticipantTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedJobParticipantTest.java index b700647c985..6237567558e 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedJobParticipantTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedJobParticipantTest.java @@ -67,6 +67,7 @@ import org.openmetadata.service.apps.bundles.searchIndex.IndexingFailureRecorder import org.openmetadata.service.cache.CacheConfig; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.search.SearchClusterMetrics; import org.openmetadata.service.search.SearchRepository; @ExtendWith(MockitoExtension.class) @@ -1028,7 +1029,9 @@ class DistributedJobParticipantTest { CollectionDAO.SearchIndexFailureDAO failureDao = mock(CollectionDAO.SearchIndexFailureDAO.class); when(collectionDAO.searchIndexFailureDAO()).thenReturn(failureDao); - when(searchRepository.createBulkSink(100, 100, 104857600L)).thenReturn(bulkSink); + when(searchRepository.createBulkSink( + 100, 100, SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES)) + .thenReturn(bulkSink); when(bulkSink.flushAndAwait(60)).thenReturn(false); try (MockedConstruction coordinatorMocked = @@ -1052,7 +1055,8 @@ class DistributedJobParticipantTest { invokeParticipantMethod( "processJobPartitions", new Class[] {SearchIndexJob.class}, runningJob); - verify(searchRepository).createBulkSink(100, 100, 104857600L); + verify(searchRepository) + .createBulkSink(100, 100, SearchClusterMetrics.DEFAULT_BULK_PAYLOAD_SIZE_BYTES); verify(bulkSink).flushAndAwait(60); assertTrue(Thread.currentThread().isInterrupted()); verify(coordinatorMocked.constructed().get(0)).claimNextPartition(jobId); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/AddUpdateLineageScriptTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/AddUpdateLineageScriptTest.java new file mode 100644 index 00000000000..768aca43188 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/AddUpdateLineageScriptTest.java @@ -0,0 +1,250 @@ +package org.openmetadata.service.search.indexes; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.openmetadata.service.search.SearchClient; + +/** + * Tests the SQL deduplication logic embedded in {@link SearchClient#ADD_UPDATE_LINEAGE}. + * + *

The Painless script runs server-side and cannot be executed in a unit test, so these tests + * implement the equivalent logic in Java. Any change to the script must be mirrored here (and + * vice versa) so regressions are caught before deployment. + * + *

The script must: + *

    + *
  1. Detect a non-empty {@code sqlQuery} on the incoming edge. + *
  2. Store the SQL text once in the doc-level {@code lineageSqlQueries} map, keyed by a + * sequential integer. + *
  3. Replace {@code sqlQuery} on the edge with a {@code sqlQueryKey} reference. + *
  4. If the same SQL already exists in the map, reuse the existing key. + *
  5. Add the edge to {@code upstreamLineage} or update the existing entry by {@code docUniqueId}. + *
+ */ +class AddUpdateLineageScriptTest { + + /** + * Java implementation of ADD_UPDATE_LINEAGE — mirrors the Painless script exactly. + * Update this whenever the script in SearchClient.java is changed. + */ + @SuppressWarnings("unchecked") + private void runScript(Map doc, Map lineageData) { + String rawSql = (String) lineageData.get("sqlQuery"); + Map edgeData; + + if (rawSql != null && !rawSql.isEmpty()) { + Map sqlMap = + (Map) + doc.computeIfAbsent("lineageSqlQueries", k -> new LinkedHashMap<>()); + + String sqlKey = null; + for (Map.Entry entry : sqlMap.entrySet()) { + if (entry.getValue().equals(rawSql)) { + sqlKey = entry.getKey(); + break; + } + } + if (sqlKey == null) { + int maxKey = 0; + for (String k : sqlMap.keySet()) { + int kInt = Integer.parseInt(k); + if (kInt > maxKey) maxKey = kInt; + } + sqlKey = String.valueOf(maxKey + 1); + sqlMap.put(sqlKey, rawSql); + } + edgeData = new HashMap<>(lineageData); + edgeData.put("sqlQueryKey", sqlKey); + edgeData.remove("sqlQuery"); + } else { + edgeData = new HashMap<>(lineageData); + } + + List> upstreamLineage = + (List>) doc.get("upstreamLineage"); + String oldSqlQueryKey = null; + boolean found = false; + for (int i = 0; i < upstreamLineage.size(); i++) { + String existingId = (String) upstreamLineage.get(i).get("docUniqueId"); + String incomingId = (String) lineageData.get("docUniqueId"); + if (existingId != null && existingId.equalsIgnoreCase(incomingId)) { + oldSqlQueryKey = (String) upstreamLineage.get(i).get("sqlQueryKey"); + upstreamLineage.set(i, edgeData); + found = true; + break; + } + } + if (!found) { + upstreamLineage.add(edgeData); + } + // Prune old SQL key if it changed and is no longer used by any edge + String newSqlQueryKey = (String) edgeData.get("sqlQueryKey"); + if (oldSqlQueryKey != null && !oldSqlQueryKey.equals(newSqlQueryKey)) { + boolean stillUsed = false; + for (Map lineage : upstreamLineage) { + if (oldSqlQueryKey.equals(lineage.get("sqlQueryKey"))) { + stillUsed = true; + break; + } + } + @SuppressWarnings("unchecked") + Map sqlMap = (Map) doc.get("lineageSqlQueries"); + if (!stillUsed && sqlMap != null) { + sqlMap.remove(oldSqlQueryKey); + } + } + } + + private Map emptyDoc() { + Map doc = new HashMap<>(); + doc.put("upstreamLineage", new ArrayList<>()); + return doc; + } + + private Map edge(String docUniqueId, String sql) { + Map edge = new HashMap<>(); + edge.put("docUniqueId", docUniqueId); + if (sql != null) { + edge.put("sqlQuery", sql); + } + return edge; + } + + // ── script constant smoke test ──────────────────────────────────────────── + + @Test + void scriptConstantContainsDedupFields() { + assertTrue( + SearchClient.ADD_UPDATE_LINEAGE.contains("lineageSqlQueries"), + "Script must reference lineageSqlQueries"); + assertTrue( + SearchClient.ADD_UPDATE_LINEAGE.contains("sqlQueryKey"), + "Script must set sqlQueryKey on the edge"); + assertTrue( + SearchClient.ADD_UPDATE_LINEAGE.contains("sqlQuery"), + "Script must read sqlQuery from the incoming edge"); + } + + // ── deduplication logic tests ───────────────────────────────────────────── + + @Test + @SuppressWarnings("unchecked") + void firstEdgeWithSql_storedInMapAndKeySet() { + Map doc = emptyDoc(); + runScript(doc, edge("edge-1", "SELECT * FROM src")); + + List> edges = (List>) doc.get("upstreamLineage"); + Map sqlMap = (Map) doc.get("lineageSqlQueries"); + + assertEquals(1, edges.size()); + assertEquals("1", edges.get(0).get("sqlQueryKey")); + assertNull(edges.get(0).get("sqlQuery"), "sqlQuery must be cleared from edge"); + assertEquals(Map.of("1", "SELECT * FROM src"), sqlMap); + } + + @Test + @SuppressWarnings("unchecked") + void secondEdgeWithSameSql_reusesKey() { + Map doc = emptyDoc(); + runScript(doc, edge("edge-1", "SELECT * FROM src")); + runScript(doc, edge("edge-2", "SELECT * FROM src")); + + List> edges = (List>) doc.get("upstreamLineage"); + Map sqlMap = (Map) doc.get("lineageSqlQueries"); + + assertEquals(2, edges.size()); + assertEquals("1", edges.get(0).get("sqlQueryKey")); + assertEquals("1", edges.get(1).get("sqlQueryKey"), "same SQL must reuse the same key"); + assertEquals(1, sqlMap.size(), "SQL stored exactly once even with 2 edges"); + } + + @Test + @SuppressWarnings("unchecked") + void edgesWithDistinctSqls_getSequentialKeys() { + Map doc = emptyDoc(); + runScript(doc, edge("edge-1", "SELECT a FROM t1")); + runScript(doc, edge("edge-2", "SELECT b FROM t2")); + runScript(doc, edge("edge-3", "SELECT c FROM t3")); + + Map sqlMap = (Map) doc.get("lineageSqlQueries"); + + assertEquals(3, sqlMap.size()); + assertEquals("SELECT a FROM t1", sqlMap.get("1")); + assertEquals("SELECT b FROM t2", sqlMap.get("2")); + assertEquals("SELECT c FROM t3", sqlMap.get("3")); + } + + @Test + @SuppressWarnings("unchecked") + void edgeWithNoSql_notModified_noMapEntry() { + Map doc = emptyDoc(); + runScript(doc, edge("edge-1", null)); + + List> edges = (List>) doc.get("upstreamLineage"); + + assertEquals(1, edges.size()); + assertNull(edges.get(0).get("sqlQueryKey"), "edge without SQL must not get a key"); + assertFalse(doc.containsKey("lineageSqlQueries"), "no SQL map created when no SQL present"); + } + + @Test + @SuppressWarnings("unchecked") + void updateExistingEdge_replacesInPlace() { + Map doc = emptyDoc(); + runScript(doc, edge("edge-1", "SELECT old FROM t")); + + Map updatedEdge = edge("edge-1", "SELECT new FROM t"); + runScript(doc, updatedEdge); + + List> edges = (List>) doc.get("upstreamLineage"); + Map sqlMap = (Map) doc.get("lineageSqlQueries"); + + assertEquals(1, edges.size(), "update must not add a second entry"); + assertEquals(1, sqlMap.size(), "old unused SQL key is pruned"); + assertEquals("2", edges.get(0).get("sqlQueryKey"), "updated edge points to new SQL key"); + assertEquals("SELECT new FROM t", sqlMap.get("2"), "map contains only the new SQL"); + } + + @Test + @SuppressWarnings("unchecked") + void batchRunScenario_660EdgesSameSql_oneMapEntry() { + String largeSql = "CREATE OR REPLACE VIEW v AS " + "SELECT id FROM src ".repeat(200); + Map doc = emptyDoc(); + + for (int i = 1; i <= 660; i++) { + runScript(doc, edge("edge-" + i, largeSql)); + } + + List> edges = (List>) doc.get("upstreamLineage"); + Map sqlMap = (Map) doc.get("lineageSqlQueries"); + + assertEquals(660, edges.size()); + assertEquals(1, sqlMap.size(), "660 identical SQLs must produce exactly 1 map entry"); + assertTrue(edges.stream().allMatch(e -> "1".equals(e.get("sqlQueryKey")))); + assertTrue(edges.stream().noneMatch(e -> e.get("sqlQuery") != null)); + } + + @Test + @SuppressWarnings("unchecked") + void mixedEdges_onlySqlEdgesDeduplicated() { + Map doc = emptyDoc(); + runScript(doc, edge("edge-1", "SELECT 1")); + runScript(doc, edge("edge-2", null)); + runScript(doc, edge("edge-3", "SELECT 1")); + + List> edges = (List>) doc.get("upstreamLineage"); + Map sqlMap = (Map) doc.get("lineageSqlQueries"); + + assertEquals(3, edges.size()); + assertEquals("1", edges.get(0).get("sqlQueryKey")); + assertNull(edges.get(1).get("sqlQueryKey")); + assertEquals("1", edges.get(2).get("sqlQueryKey")); + assertEquals(1, sqlMap.size()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java index 2e1c37746fc..bfd46e28dea 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/SearchIndexTest.java @@ -3,12 +3,15 @@ package org.openmetadata.service.search.indexes; import static org.junit.jupiter.api.Assertions.*; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.junit.jupiter.api.Test; +import org.openmetadata.schema.api.lineage.EsLineageData; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.util.FullyQualifiedName; @@ -280,6 +283,154 @@ class SearchIndexTest { } } + // ── SQL deduplication tests ────────────────────────────────────────────── + + @Test + void testDeduplicateSql_noEdges_returnsEmptyMap() { + Map result = + SearchIndexUtils.deduplicateSqlAcrossEdges(Collections.emptyList()); + assertTrue(result.isEmpty()); + } + + @Test + void testDeduplicateSql_edgesWithNoSql_untouched() { + EsLineageData e1 = new EsLineageData(); + EsLineageData e2 = new EsLineageData(); + + Map result = SearchIndexUtils.deduplicateSqlAcrossEdges(List.of(e1, e2)); + + assertTrue(result.isEmpty(), "no SQL means no dedup map entries"); + assertNull(e1.getSqlQueryKey()); + assertNull(e2.getSqlQueryKey()); + } + + @Test + void testDeduplicateSql_singleEdgeWithSql_getsKeyOne() { + EsLineageData edge = new EsLineageData().withSqlQuery("SELECT 1"); + + Map result = SearchIndexUtils.deduplicateSqlAcrossEdges(List.of(edge)); + + assertEquals(Map.of("1", "SELECT 1"), result); + assertEquals("1", edge.getSqlQueryKey()); + assertNull(edge.getSqlQuery(), "sql text should be cleared after keying"); + } + + @Test + void testDeduplicateSql_identicalSqlAcrossEdges_sameKey() { + String sql = "CREATE OR REPLACE VIEW analytics AS SELECT * FROM source"; + List edges = + List.of( + new EsLineageData().withSqlQuery(sql), + new EsLineageData().withSqlQuery(sql), + new EsLineageData().withSqlQuery(sql)); + + Map result = SearchIndexUtils.deduplicateSqlAcrossEdges(edges); + + assertEquals(1, result.size(), "identical SQL stored exactly once"); + assertEquals("1", result.keySet().iterator().next()); + for (EsLineageData edge : edges) { + assertEquals("1", edge.getSqlQueryKey(), "all edges should reference the same key"); + assertNull(edge.getSqlQuery(), "sql text cleared on all edges"); + } + } + + @Test + void testDeduplicateSql_distinctSqls_getSequentialKeys() { + EsLineageData e1 = new EsLineageData().withSqlQuery("SELECT a FROM t1"); + EsLineageData e2 = new EsLineageData().withSqlQuery("SELECT b FROM t2"); + EsLineageData e3 = new EsLineageData().withSqlQuery("SELECT c FROM t3"); + + Map result = SearchIndexUtils.deduplicateSqlAcrossEdges(List.of(e1, e2, e3)); + + assertEquals(3, result.size()); + assertEquals("SELECT a FROM t1", result.get("1")); + assertEquals("SELECT b FROM t2", result.get("2")); + assertEquals("SELECT c FROM t3", result.get("3")); + assertEquals("1", e1.getSqlQueryKey()); + assertEquals("2", e2.getSqlQueryKey()); + assertEquals("3", e3.getSqlQueryKey()); + } + + @Test + void testDeduplicateSql_mixedEdgesSomeSqlSomeNot() { + String sql = "SELECT id FROM src"; + EsLineageData withSql1 = new EsLineageData().withSqlQuery(sql); + EsLineageData noSql = new EsLineageData(); + EsLineageData withSql2 = new EsLineageData().withSqlQuery(sql); + + Map result = + SearchIndexUtils.deduplicateSqlAcrossEdges(List.of(withSql1, noSql, withSql2)); + + assertEquals(Map.of("1", sql), result); + assertEquals("1", withSql1.getSqlQueryKey()); + assertNull(withSql1.getSqlQuery()); + assertNull(noSql.getSqlQueryKey(), "edge without SQL should not get a key"); + assertEquals("1", withSql2.getSqlQueryKey()); + } + + @Test + void testDeduplicateSql_batchRunScenario_660EdgesSameSql() { + // Mirrors the real-world scenario: a BATCH_RUN VIEW has 660+ upstream tables, + // each edge carrying the same ~30 KB CREATE VIEW SQL. + // After dedup the map should have exactly 1 entry and all edges share key "1". + String largeSql = + "CREATE OR REPLACE VIEW batch_view AS " + "SELECT * FROM source_table ".repeat(500); + int edgeCount = 660; + + List edges = + IntStream.range(0, edgeCount) + .mapToObj(i -> new EsLineageData().withSqlQuery(largeSql)) + .collect(Collectors.toList()); + + Map result = SearchIndexUtils.deduplicateSqlAcrossEdges(edges); + + assertEquals(1, result.size(), "660 identical SQLs deduplicated to 1 entry"); + assertEquals(largeSql, result.get("1")); + for (EsLineageData edge : edges) { + assertEquals("1", edge.getSqlQueryKey()); + assertNull(edge.getSqlQuery()); + } + } + + @Test + void testDeduplicateSql_partialDuplication_correctGrouping() { + // sqlA appears on 3 edges, sqlB appears on 2 edges, sqlC appears once. + String sqlA = "SELECT a FROM tA"; + String sqlB = "SELECT b FROM tB"; + String sqlC = "SELECT c FROM tC"; + + List edges = + List.of( + new EsLineageData().withSqlQuery(sqlA), + new EsLineageData().withSqlQuery(sqlB), + new EsLineageData().withSqlQuery(sqlA), + new EsLineageData().withSqlQuery(sqlC), + new EsLineageData().withSqlQuery(sqlA), + new EsLineageData().withSqlQuery(sqlB)); + + Map result = SearchIndexUtils.deduplicateSqlAcrossEdges(edges); + + assertEquals(3, result.size()); + // Keys assigned in first-seen order + String keyA = edges.get(0).getSqlQueryKey(); + String keyB = edges.get(1).getSqlQueryKey(); + String keyC = edges.get(3).getSqlQueryKey(); + + assertNotEquals(keyA, keyB); + assertNotEquals(keyA, keyC); + assertNotEquals(keyB, keyC); + + // All edges with the same SQL share the same key + assertEquals(keyA, edges.get(2).getSqlQueryKey()); + assertEquals(keyA, edges.get(4).getSqlQueryKey()); + assertEquals(keyB, edges.get(5).getSqlQueryKey()); + + // The map stores the correct SQL for each key + assertEquals(sqlA, result.get(keyA)); + assertEquals(sqlB, result.get(keyB)); + assertEquals(sqlC, result.get(keyC)); + } + private Map buildDocWithChangeDescription(Object newValue) { Map fieldChange = new HashMap<>(); fieldChange.put("name", "deleted"); diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/ai_agent_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/ai_agent_index_mapping.json index ad1d3bb7767..bbc2b9090ef 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/ai_agent_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/ai_agent_index_mapping.json @@ -694,6 +694,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/ai_governance_policy_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/ai_governance_policy_index_mapping.json index 7e64c90e387..c9a2b9fc72a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/ai_governance_policy_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/ai_governance_policy_index_mapping.json @@ -702,6 +702,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/api_collection_index_mapping.json index dcbbad55ba0..15f86483f29 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/api_collection_index_mapping.json @@ -615,6 +615,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -731,6 +734,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json index a6abb39cde7..a1487bdb631 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/api_endpoint_index_mapping.json @@ -786,6 +786,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -955,6 +958,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/chart_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/chart_index_mapping.json index c0919bea5c2..bb0518c870e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/chart_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/chart_index_mapping.json @@ -276,6 +276,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -682,6 +685,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/container_index_mapping.json index 3486f458db2..5a1975a865d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/container_index_mapping.json @@ -269,6 +269,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -936,6 +939,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json index 548a060be9f..039d8cf0b28 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json @@ -488,6 +488,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -790,6 +793,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_index_mapping.json index 511234db039..7d4af72c6fa 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/dashboard_index_mapping.json @@ -259,6 +259,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -823,6 +826,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/directory_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/directory_index_mapping.json index 863cefd2b84..7f8da963da1 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/directory_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/directory_index_mapping.json @@ -646,6 +646,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -772,6 +775,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/file_index_mapping.json index dff0c9df151..da08b918c80 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/file_index_mapping.json @@ -698,6 +698,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -824,6 +827,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/llm_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/llm_model_index_mapping.json index 7e64c90e387..c9a2b9fc72a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/llm_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/llm_model_index_mapping.json @@ -702,6 +702,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/metric_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/metric_index_mapping.json index 1cf0ce13957..a3960d8ce0d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/metric_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/metric_index_mapping.json @@ -592,6 +592,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -697,6 +700,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json index 784d30c9b53..81a67db7895 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/mlmodel_index_mapping.json @@ -259,6 +259,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -831,6 +834,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/pipeline_index_mapping.json index 5cf766b022e..cb3039e39d6 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/pipeline_index_mapping.json @@ -304,6 +304,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -738,6 +741,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/prompt_template_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/prompt_template_index_mapping.json index 7e64c90e387..c9a2b9fc72a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/prompt_template_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/prompt_template_index_mapping.json @@ -702,6 +702,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/search_entity_index_mapping.json index 2d60398fd39..b6d4607d637 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/search_entity_index_mapping.json @@ -399,6 +399,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -793,6 +796,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/spreadsheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/spreadsheet_index_mapping.json index ae5471f5a55..dfa27e381d4 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/spreadsheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/spreadsheet_index_mapping.json @@ -687,6 +687,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -813,6 +816,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json index 18c91897a43..4312342107b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/stored_procedure_index_mapping.json @@ -695,6 +695,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -811,6 +814,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/table_index_mapping.json index 0c9972b2987..653c2a69ac9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/table_index_mapping.json @@ -844,6 +844,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -1055,6 +1058,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/team_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/team_index_mapping.json index f848dba05ef..90debfcc9e7 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/team_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/team_index_mapping.json @@ -219,41 +219,7 @@ } }, "users": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } + "enabled": false }, "userCount": { "type": "long" @@ -296,41 +262,7 @@ } }, "defaultRoles": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } + "enabled": false }, "isJoinable": { "type": "text" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/topic_index_mapping.json index 73ddd11bf64..89e24dc7933 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/topic_index_mapping.json @@ -649,6 +649,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -818,6 +821,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/en/worksheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/en/worksheet_index_mapping.json index 795154504b8..f15cbe4adec 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/en/worksheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/en/worksheet_index_mapping.json @@ -811,6 +811,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -937,6 +940,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_agent_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_agent_index_mapping.json index 3d08a3144e4..82fdf3c84e2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_agent_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_agent_index_mapping.json @@ -699,6 +699,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_governance_policy_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_governance_policy_index_mapping.json index 257c3608b6e..e9dff9b2b82 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_governance_policy_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/ai_governance_policy_index_mapping.json @@ -707,6 +707,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json index b95e3692402..4d5051c8f21 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json @@ -594,6 +594,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -704,6 +707,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json index 17e874e2042..440fc74f080 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json @@ -773,6 +773,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -918,6 +921,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json index f9c66f90f4a..347ab1af12b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json @@ -253,6 +253,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -710,6 +713,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json index c38914b0ed9..0f2544c8f25 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json @@ -256,6 +256,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -858,6 +861,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json index 55852422ee2..9a3989b0fde 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json @@ -254,6 +254,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -743,6 +746,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json index 8a9c30f008f..04591366644 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json @@ -256,6 +256,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -805,6 +808,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/directory_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/directory_index_mapping.json index 90f8ee00b57..8ada8865817 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/directory_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/directory_index_mapping.json @@ -678,6 +678,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -754,6 +757,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json index a181da9dc36..183e6f74a6b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json @@ -693,6 +693,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -769,6 +772,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/llm_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/llm_model_index_mapping.json index 257c3608b6e..e9dff9b2b82 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/llm_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/llm_model_index_mapping.json @@ -707,6 +707,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json index afc4d824646..1dd076b33d7 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json @@ -588,6 +588,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -681,6 +684,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json index 3a7606a39cc..7f8740b8431 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json @@ -255,6 +255,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -807,6 +810,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json index 58147475a00..86ce8acc928 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json @@ -302,6 +302,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -695,6 +698,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/prompt_template_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/prompt_template_index_mapping.json index 257c3608b6e..e9dff9b2b82 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/prompt_template_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/prompt_template_index_mapping.json @@ -707,6 +707,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json index 7bc97c2105e..9813ccb8f5b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json @@ -594,6 +594,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -780,6 +783,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/spreadsheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/spreadsheet_index_mapping.json index 0d31c42e550..6f05ba4d722 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/spreadsheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/spreadsheet_index_mapping.json @@ -678,6 +678,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -754,6 +757,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json index 025a13cfec7..2865cb3025f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json @@ -771,6 +771,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -887,6 +890,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json index 55bb2ea9bec..66317765b5d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json @@ -686,6 +686,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -1036,6 +1039,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/team_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/team_index_mapping.json index ef60cbd985c..38ddf70819f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/team_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/team_index_mapping.json @@ -211,41 +211,7 @@ } }, "users": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } + "enabled": false }, "userCount": { "type": "long" @@ -288,41 +254,7 @@ } }, "defaultRoles": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } + "enabled": false }, "isJoinable": { "type": "text" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json index ca4235accfb..8171f4071ae 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json @@ -255,6 +255,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -790,6 +793,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/worksheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/worksheet_index_mapping.json index e66809c9f90..30be5b3cc15 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/worksheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/worksheet_index_mapping.json @@ -751,6 +751,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -827,6 +830,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_agent_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_agent_index_mapping.json index 10dacf1addb..19703c42502 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_agent_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_agent_index_mapping.json @@ -713,6 +713,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_governance_policy_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_governance_policy_index_mapping.json index f521a039c9b..a0501dede4d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_governance_policy_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/ai_governance_policy_index_mapping.json @@ -721,6 +721,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json index dd619e7ff12..e0cd68dbc6e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json @@ -632,6 +632,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -748,6 +751,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json index 33db1f08346..7bb200e7628 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json @@ -803,6 +803,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -972,6 +975,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/chart_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/chart_index_mapping.json index c66db89d2de..350d332333d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/chart_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/chart_index_mapping.json @@ -295,6 +295,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -699,6 +702,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json index cc0fe1165d6..8010edf3536 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json @@ -279,6 +279,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -909,6 +912,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json index 2e267e13375..8145399e3a5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json @@ -505,6 +505,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -762,6 +765,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json index 827140bf35e..00e356ed467 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json @@ -278,6 +278,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -840,6 +843,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/directory_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/directory_index_mapping.json index 94b8de1c780..2c9b3850f05 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/directory_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/directory_index_mapping.json @@ -579,6 +579,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -655,6 +658,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json index 2a64fe42bae..3ad44845637 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json @@ -634,6 +634,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -710,6 +713,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/llm_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/llm_model_index_mapping.json index f521a039c9b..a0501dede4d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/llm_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/llm_model_index_mapping.json @@ -721,6 +721,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/metric_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/metric_index_mapping.json index 29a215b2fcd..5f60aa7af3a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/metric_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/metric_index_mapping.json @@ -562,6 +562,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -667,6 +670,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json index 9c721a27941..908058f2845 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json @@ -278,6 +278,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -848,6 +851,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json index 8fa47edcd06..16a5f719e4b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json @@ -322,6 +322,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -755,6 +758,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/prompt_template_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/prompt_template_index_mapping.json index f521a039c9b..a0501dede4d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/prompt_template_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/prompt_template_index_mapping.json @@ -721,6 +721,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json index 5b278c1abfa..69c579ef390 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json @@ -421,6 +421,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -810,6 +813,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/spreadsheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/spreadsheet_index_mapping.json index 93fe96dcbbe..a7566be8076 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/spreadsheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/spreadsheet_index_mapping.json @@ -633,6 +633,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -709,6 +712,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json index 88e24a4cf76..513535becf4 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json @@ -712,6 +712,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -828,6 +831,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json index 43d14622944..28a1da38a15 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json @@ -852,6 +852,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -1042,6 +1045,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/team_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/team_index_mapping.json index 69cc9984888..20c790a5822 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/team_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/team_index_mapping.json @@ -238,41 +238,7 @@ } }, "users": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } + "enabled": false }, "userCount": { "type": "long" @@ -315,41 +281,7 @@ } }, "defaultRoles": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } + "enabled": false }, "isJoinable": { "type": "text" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json index 915427b8bb2..9e4181f99b8 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json @@ -666,6 +666,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -835,6 +838,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/worksheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/worksheet_index_mapping.json index 6ed09a90f99..63c9e7ab6ac 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/worksheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/worksheet_index_mapping.json @@ -693,6 +693,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -769,6 +772,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_agent_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_agent_index_mapping.json index e6c8f154099..f9e78c0f364 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_agent_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_agent_index_mapping.json @@ -689,6 +689,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_governance_policy_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_governance_policy_index_mapping.json index 1d558e753a1..bedc59536ab 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_governance_policy_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/ai_governance_policy_index_mapping.json @@ -697,6 +697,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json index 5c3c172b475..ca53dadfd1a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json @@ -588,6 +588,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -704,6 +707,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json index 6dd7f54700a..db1b6d4e9de 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json @@ -774,6 +774,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -919,6 +922,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/chart_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/chart_index_mapping.json index 364055c50e7..5e00e8feb94 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/chart_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/chart_index_mapping.json @@ -253,6 +253,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -696,6 +699,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json index 8dd00025225..a28d9551797 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json @@ -260,6 +260,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -860,6 +863,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json index cc23ca872cd..eec97c2bdd0 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json @@ -318,6 +318,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -741,6 +744,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json index 6cf14576278..060113e5c72 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json @@ -256,6 +256,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -762,6 +765,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/directory_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/directory_index_mapping.json index 57b846f2e6f..c9119fe16ef 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/directory_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/directory_index_mapping.json @@ -651,6 +651,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -727,6 +730,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json index cce32121196..5b9e0605d06 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json @@ -666,6 +666,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -742,6 +745,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/llm_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/llm_model_index_mapping.json index 1d558e753a1..bedc59536ab 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/llm_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/llm_model_index_mapping.json @@ -697,6 +697,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/metric_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/metric_index_mapping.json index fdfc7e34232..f78ef1ae375 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/metric_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/metric_index_mapping.json @@ -592,6 +592,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -673,6 +676,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json index 7e181cac261..0d697f444f9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json @@ -253,6 +253,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -801,6 +804,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json index 9b86f1c7d28..7828c2c1cb8 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json @@ -303,6 +303,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -698,6 +701,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/prompt_template_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/prompt_template_index_mapping.json index 1d558e753a1..bedc59536ab 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/prompt_template_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/prompt_template_index_mapping.json @@ -697,6 +697,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json index f7dc94c4408..f3be0b9ab87 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json @@ -453,6 +453,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -764,6 +767,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/spreadsheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/spreadsheet_index_mapping.json index b5039b2f171..e15b4c52d97 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/spreadsheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/spreadsheet_index_mapping.json @@ -651,6 +651,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -727,6 +730,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json index bbc0829bd21..804b2d40a09 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json @@ -771,6 +771,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -887,6 +890,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json index 227e47d7677..08e64524f9c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json @@ -258,6 +258,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -1027,6 +1030,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/team_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/team_index_mapping.json index a79be297846..a2acb2cf10a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/team_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/team_index_mapping.json @@ -201,41 +201,7 @@ } }, "users": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } + "enabled": false }, "userCount": { "type": "long" @@ -278,41 +244,7 @@ } }, "defaultRoles": { - "properties": { - "id": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 36 - } - } - }, - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "fullyQualifiedName": { - "type": "text" - }, - "description": { - "type": "text" - }, - "deleted": { - "type": "boolean" - }, - "href": { - "type": "text" - } - } + "enabled": false }, "isJoinable": { "type": "text" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json index 6b38f31bd48..61696fad2c2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json @@ -604,6 +604,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -741,6 +744,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/worksheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/worksheet_index_mapping.json index c82e9c87164..24d3a959be7 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/worksheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/worksheet_index_mapping.json @@ -726,6 +726,9 @@ "ignore_above": 512 } } + }, + "sqlQueryKey": { + "type": "keyword" } } }, @@ -802,6 +805,10 @@ "ownerName": { "type": "keyword", "normalizer": "lowercase_normalizer" + }, + "lineageSqlQueries": { + "type": "object", + "enabled": false } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/lineage/esLineageData.json b/openmetadata-spec/src/main/resources/json/schema/api/lineage/esLineageData.json index 7c80c1c245f..bda8d5cb995 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/lineage/esLineageData.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/lineage/esLineageData.json @@ -45,6 +45,10 @@ "description": "Sql Query associated.", "type": "string" }, + "sqlQueryKey": { + "description": "Key referencing the full SQL text in the parent document's lineageSqlQueries map. Set when the same SQL appears in multiple edges to avoid storing it repeatedly. Look up the actual SQL in lineageSqlQueries[sqlQueryKey].", + "type": "string" + }, "columns": { "description": "Columns associated.", "type": "array", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json index 4f7ca974f75..1239a88811f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/searchIndexingAppConfig.json @@ -42,7 +42,7 @@ "description": "Maximum number of events sent in a batch (Default 100).", "type": "integer", "existingJavaType": "java.lang.Long", - "default": 104857600 + "default": 9437184 }, "producerThreads": { "title": "Number of Producer Threads", diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json index 582f87f4297..347c1be6973 100644 --- a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json +++ b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json @@ -154,7 +154,7 @@ "description": "Payload size in bytes depending on config.", "type": "integer", "existingJavaType": "java.lang.Long", - "default": 104857600, + "default": 9437184, "minimum": 1 }, "producerThreads": { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/lineage/esLineageData.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/lineage/esLineageData.ts index f0fd1e484a5..6cc5c94e0aa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/lineage/esLineageData.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/lineage/esLineageData.ts @@ -63,6 +63,12 @@ export interface EsLineageData { * Sql Query associated. */ sqlQuery?: string; + /** + * Key referencing the full SQL text in the parent document's lineageSqlQueries map. Set + * when the same SQL appears in multiple edges to avoid storing it repeatedly. Look up the + * actual SQL in lineageSqlQueries[sqlQueryKey]. + */ + sqlQueryKey?: string; /** * Lineage path through temporary/intermediate tables. Each element represents a hop with * fromEntity and toEntity fields.