mirror of
https://github.com/open-metadata/OpenMetadata
synced 2026-05-24 09:39:11 +00:00
cf0fa0a519
* Openlineage Kinesis Support * Update generated TypeScript types * marking field as required * test-connection name improvement * pagination improvement * test-connection name improvement * Update generated TypeScript types * nested broker-config migration file * newline added to yaml * Migration to 1.11.2 * Migration to 1.11.12* * fix: add throttle mechanism to kinesis get_records loop Co-authored-by: Khairajani <Khairajani@users.noreply.github.com> * fix: prevent timeout reset on sequential shard polling Co-authored-by: Khairajani <Khairajani@users.noreply.github.com> * Kinesis test-case * Kinesis test-case * setting lineageInformation object model and not raw dict --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Gitar <noreply@gitar.ai> Co-authored-by: Khairajani <Khairajani@users.noreply.github.com>
26 lines
1 KiB
SQL
26 lines
1 KiB
SQL
-- Migrate OpenLineage connection config: move Kafka-specific fields into brokerConfig
|
|
-- Supports the schema change adding Kinesis as an alternative broker type
|
|
UPDATE pipeline_service_entity
|
|
SET json = JSON_SET(
|
|
JSON_REMOVE(json,
|
|
'$.connection.config.brokersUrl',
|
|
'$.connection.config.topicName',
|
|
'$.connection.config.consumerGroupName',
|
|
'$.connection.config.consumerOffsets',
|
|
'$.connection.config.poolTimeout',
|
|
'$.connection.config.sessionTimeout',
|
|
'$.connection.config.securityProtocol',
|
|
'$.connection.config.sslConfig',
|
|
'$.connection.config.saslConfig'
|
|
),
|
|
'$.connection.config.brokerConfig',
|
|
JSON_REMOVE(
|
|
JSON_EXTRACT(json, '$.connection.config'),
|
|
'$.type',
|
|
'$.pipelineFilterPattern',
|
|
'$.supportsMetadataExtraction'
|
|
)
|
|
)
|
|
WHERE serviceType = 'OpenLineage'
|
|
AND JSON_CONTAINS_PATH(json, 'one', '$.connection.config.brokersUrl')
|
|
AND NOT JSON_CONTAINS_PATH(json, 'one', '$.connection.config.brokerConfig');
|