mirror of
https://github.com/open-metadata/OpenMetadata
synced 2026-05-24 09:39:11 +00:00
5e1416447f
* fix(sampler): respect randomizedSample flag at 100% percentage sampling When profileSample is 100% with PERCENTAGE type, the sampler short-circuits and returns the raw dataset without any randomization, even when randomizedSample is True (the default). Split the combined condition so: - No profileSample set -> return raw dataset (no sampling configured) - 100% PERCENTAGE + randomizedSample=False -> return raw dataset (optimization) - 100% PERCENTAGE + randomizedSample=True -> go through normal sampling path which applies RandomNumFn/df.sample for proper row shuffling Fixes #21304 * Address review: use 'is False' for Optional[bool] and add unit tests - Fix randomizedSample check from 'not' to 'is False' in both SQASampler and DatalakeSampler to correctly handle None (Optional[bool] default=True) - Add unit tests verifying 100%% PERCENTAGE behavior for randomizedSample values True, False, and None * Add ORDER BY on random column in fetch_sample_data for true randomization The get_dataset() fix ensures 100% PERCENTAGE + randomizedSample routes through get_sample_query() which produces a CTE with a random column. Now fetch_sample_data() detects that column and applies ORDER BY before LIMIT, so each call returns a different subset of rows. Also add real-DB integration tests using SQLite for the 100% PERCENTAGE edge case (True, False, None). * Address review: remove stale comment, unused import, add return assertions * Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Address review: move ORDER BY to get_sample_query, clean up fetch_sample_data - Move ORDER BY rnd.c.random into get_sample_query() PERCENTAGE branch, gated on randomizedSample is not False (mirrors ABSOLUTE branch pattern) - Revert fetch_sample_data() to original: remove ds_columns variable, random_column detection, and ORDER BY logic (ordering now handled in CTE) - Remove duplicate assertions in DatalakeSampler100Pct tests * Address review: None defaults to False for randomizedSample Per TeddyCr's feedback, randomization is computationally heavy and should not be the default. Changed from 'is False'/'is not False' to truthiness checks so None (unset) behaves the same as False. Only explicit randomizedSample=True triggers ORDER BY and skips the 100% fast path. This is consistent with the ABSOLUTE branch which already uses truthiness checks. * Fix integration test: None should skip sample_query (matches truthiness semantics) * fix(tests): update BigQuery view sampling expected queries with ORDER BY BigQuery views fall through to SQASampler.get_sample_query() which now adds ORDER BY rnd.random when randomizedSample is enabled. Update the expected SQL strings in test_sampling_for_views and test_sampling_view_with_partition to match. * refactor: use explicit is False for randomizedSample checks Address review comments: SampleConfig.randomizedSample defaults to True, so only an explicit False should disable randomization. Using is False / is not False instead of truthiness ensures None follows the model default (enabled) rather than being incorrectly treated as disabled. * ci: re-trigger checks after SIGSEGV flake * refactor: only explicit True randomizes, add non-determinism tests * test: increase non-determinism iterations to reduce flakiness * chore: added randomize as false * fix: align randomizedSample defaults with schema (false) * fix: remove ORDER BY from BigQuery test expectations BigQuery sampling tests create SampleConfig without setting randomizedSample, which now defaults to False. Since ORDER BY is only added when randomizedSample is True, the expected query strings should not include ORDER BY. Also fix inaccurate docstring in test_sample.py. * test: increase non-determinism test iterations to reduce flakiness Increase fetch_sample_data loop from 10 to 20 iterations to further reduce the theoretical probability of a false failure in the randomized ordering test. --------- Co-authored-by: Teddy <teddy.crepineau@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
58 lines
No EOL
2.5 KiB
SQL
58 lines
No EOL
2.5 KiB
SQL
UPDATE ingestion_pipeline_entity
|
|
SET json = (json::jsonb #- '{sourceConfig,config,computeMetrics}')::json
|
|
WHERE json::jsonb -> 'sourceConfig' -> 'config' -> 'computeMetrics' IS NOT NULL
|
|
AND pipelineType = 'profiler';
|
|
|
|
-- Set randomizedSample to false where it was true (old default behavior)
|
|
UPDATE ingestion_pipeline_entity
|
|
SET json = jsonb_set(json::jsonb, '{sourceConfig,config,randomizedSample}', 'false'::jsonb)::json
|
|
WHERE json::jsonb #>> '{sourceConfig,config,randomizedSample}' = 'true'
|
|
AND pipelineType = 'profiler';
|
|
|
|
UPDATE table_entity
|
|
SET json = jsonb_set(json::jsonb, '{tableProfilerConfig,randomizedSample}', 'false'::jsonb)::json
|
|
WHERE json::jsonb #>> '{tableProfilerConfig,randomizedSample}' = 'true';
|
|
|
|
UPDATE database_entity
|
|
SET json = jsonb_set(json::jsonb, '{databaseProfilerConfig,randomizedSample}', 'false'::jsonb)::json
|
|
WHERE json::jsonb #>> '{databaseProfilerConfig,randomizedSample}' = 'true';
|
|
|
|
UPDATE database_schema_entity
|
|
SET json = jsonb_set(json::jsonb, '{databaseSchemaProfilerConfig,randomizedSample}', 'false'::jsonb)::json
|
|
WHERE json::jsonb #>> '{databaseSchemaProfilerConfig,randomizedSample}' = 'true';
|
|
|
|
-- Hard-delete ingestion pipelines for Iceberg services (must run before service migration)
|
|
DELETE FROM ingestion_pipeline_entity ipe
|
|
USING dbservice_entity dse
|
|
WHERE dse.serviceType = 'Iceberg'
|
|
AND ipe.json::jsonb -> 'service' ->> 'type' = 'databaseService'
|
|
AND ipe.json::jsonb -> 'service' ->> 'id' = dse.id;
|
|
|
|
-- Migrate Iceberg database services to CustomDatabase (connector removed)
|
|
-- serviceType is a GENERATED column derived from json, so only update json
|
|
UPDATE dbservice_entity
|
|
SET json = jsonb_set(
|
|
jsonb_set(
|
|
json::jsonb,
|
|
'{serviceType}', '"CustomDatabase"'
|
|
),
|
|
'{connection,config,type}', '"CustomDatabase"'
|
|
)::json
|
|
WHERE serviceType = 'Iceberg';
|
|
|
|
-- Migrate serviceType in child entities (serviceType is in JSON blob only, no generated column)
|
|
UPDATE database_entity
|
|
SET json = jsonb_set(json::jsonb, '{serviceType}', '"CustomDatabase"')::json
|
|
WHERE json->>'serviceType' = 'Iceberg';
|
|
|
|
UPDATE database_schema_entity
|
|
SET json = jsonb_set(json::jsonb, '{serviceType}', '"CustomDatabase"')::json
|
|
WHERE json->>'serviceType' = 'Iceberg';
|
|
|
|
UPDATE table_entity
|
|
SET json = jsonb_set(json::jsonb, '{serviceType}', '"CustomDatabase"')::json
|
|
WHERE json->>'serviceType' = 'Iceberg';
|
|
|
|
UPDATE stored_procedure_entity
|
|
SET json = jsonb_set(json::jsonb, '{serviceType}', '"CustomDatabase"')::json
|
|
WHERE json->>'serviceType' = 'Iceberg'; |