mirror of
https://github.com/open-metadata/OpenMetadata
synced 2026-05-24 09:39:11 +00:00
5009059441
* chore(ingestion): replace black/isort/pycln with ruff - Swap formatter + import-sorter + unused-import tooling for ruff (line-length 120, target py3.10) in ingestion + openmetadata-airflow-apis - Drop dead [tool.mypy] config; basedpyright is the active type checker - Bump requires-python to >=3.10 to match noxfile and CLAUDE.md (3.9 is documented as broken on Mac in noxfile.py) - Bump pre-commit-hooks v2.3 -> v5.0; the new check-json catches four pre-existing JSON issues now excluded with an inline TODO - Update Makefile py_format / py_format_check targets to call ruff * chore(ingestion): grandfather ruff lint violations and apply ruff format - 253 noqa markers added via 'ruff check --add-noqa' across 128 files, freezing existing violations so this PR is a tooling-only swap. Per-rule cleanup tracked in the TODO comment in ingestion/pyproject.toml. - Bulk reformat from black 22.3 -> ruff format @ line-length 120. Cosmetic only: imports balanced (-32/+32), structural keywords balanced (-2221/+2221), no logic changes. - Star-import rules (F403/F405) globally ignored; refactoring wildcard imports across connectors is a separate effort. * chore(ingestion): fix pylint findings surfaced after ruff format - filters.py: drop redundant parens around re.match(...) in `if` (C0325 superfluous-parens) — exposed when ruff format unwrapped them - nosql_adaptor.py: move `# pylint: disable=unused-argument` from the `column:` line to the `def` line so it covers `table` too (W0613) — scope was line-based, lost when ruff split params onto multiple lines - action1xx.py: replace `arguments-differ` with `signature-differs` in the disable directive (was always wrong code) and drop the now-useless `unused-argument` suppression (I0021) * fix(ingestion): make ruff extend-exclude robust to multi-root invocations CI's `make py_format_check` runs from the repo root and passes both `ingestion/` and `./openmetadata-airflow-apis/` to ruff in a single invocation. With multiple root paths, ruff's parallel file discovery races on extend-exclude matching against the project root, so files under `ingestion/src/metadata/generated/` were intermittently scanned and produced ~830 I001 violations. 20-run repro: 10/20 fail without the fix, 20/20 pass with the fix. Each excluded directory now appears twice in extend-exclude: - the project-root-relative pattern (cwd = ingestion/) - the prefixed pattern (cwd = repo root, multi-root invocation) * chore(ingestion): address gitar-bot findings + cross-version pylint disable - openmetadata-airflow-apis/pyproject.toml: switch coverage to module-name source + [tool.coverage.paths] glob remap (matches the ingestion pattern). Drops the hardcoded `env/lib/python3.9/site-packages/...` source path, which broke after the requires-python bump to 3.10. (Finding 1) - ingestion/setup.py: remove dead python_version<'3.9' / >='3.9' guards on mysql-connector-python and testcontainers; promote locust to a regular test dep (was conditionally added under sys.version_info >= (3, 9)). Also remove the now-unused `import sys`. (Finding 3) - ingestion/src/metadata/great_expectations/action1xx.py: cover both arguments-differ (great_expectations 0.18.x parent) and signature-differs (great_expectations 1.x parent) in the pylint disable comment, since CI installs 0.18.x and local often has 1.x. unused-argument covers the unused action_context. The opposite rule fires as I0021 useless-suppression on each environment, which is informational and does not affect pylint's exit code.
224 lines
8.4 KiB
Python
224 lines
8.4 KiB
Python
# Copyright 2025 Collate
|
|
# Licensed under the Collate Community License, Version 1.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""
|
|
Test Entity Link build behavior
|
|
"""
|
|
|
|
from unittest import TestCase
|
|
|
|
from metadata.generated.schema.entity.data.dashboard import Dashboard
|
|
from metadata.generated.schema.entity.data.table import Table
|
|
from metadata.utils import entity_link
|
|
from metadata.utils.entity_link import get_entity_link
|
|
|
|
|
|
class TestEntityLink(TestCase):
|
|
"""
|
|
Validate EntityLink building
|
|
"""
|
|
|
|
def test_split(self):
|
|
this = self
|
|
|
|
class EntityLinkTest:
|
|
"""
|
|
Test helper class
|
|
"""
|
|
|
|
def __init__(self, entitylink, split_list):
|
|
self.entitylink = entitylink
|
|
self.split_list = split_list
|
|
|
|
def validate(self, fn_resp, check_split):
|
|
this.assertEqual(fn_resp, check_split)
|
|
this.assertEqual(len(fn_resp), len(check_split))
|
|
|
|
xs = [
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog1>",
|
|
["table", "bigquery_gcp.shopify.raw_product_catalog1"],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog2::description>",
|
|
["table", "bigquery_gcp.shopify.raw_product_catalog2", "description"],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog3::columns::comment>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog3",
|
|
"columns",
|
|
"comment",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::ingestionPipeline::fivetran_gcp.shopify.raw_product_catalog3>",
|
|
[
|
|
"ingestionPipeline",
|
|
"fivetran_gcp.shopify.raw_product_catalog3",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog4::columns::comment::description>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog4",
|
|
"columns",
|
|
"comment",
|
|
"description",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::database::bigquery_gcp.shopify>",
|
|
["database", "bigquery_gcp.shopify"],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::database::bigquery_gcp.shopify::tags>",
|
|
["database", "bigquery_gcp.shopify", "tags"],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw-product-catalog5>",
|
|
["table", "bigquery_gcp.shopify.raw-product-catalog5"],
|
|
),
|
|
EntityLinkTest(
|
|
'<#E::table::bigquery_gcp.shopify."raw-product-catalog6"::description>',
|
|
[
|
|
"table",
|
|
'bigquery_gcp.shopify."raw-product-catalog6"',
|
|
"description",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw-product-catalog5::description>",
|
|
["table", "bigquery_gcp.shopify.raw-product-catalog5", "description"],
|
|
),
|
|
EntityLinkTest(
|
|
'<#E::table::bigquery_gcp."shop-ify"."raw-product-catalog6">',
|
|
["table", 'bigquery_gcp."shop-ify"."raw-product-catalog6"'],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::随机的>",
|
|
["table", "随机的"],
|
|
),
|
|
EntityLinkTest(
|
|
'<#E::table::ExampleWithFolder.withfolder.examplewithfolder."folderpath/username.csv">',
|
|
[
|
|
"table",
|
|
'ExampleWithFolder.withfolder.examplewithfolder."folderpath/username.csv"',
|
|
],
|
|
),
|
|
# Test ENTITY_TYPE keywords as column names
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::topic>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"topic",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::user>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"user",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::database>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"database",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::role>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"role",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::chart>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"chart",
|
|
],
|
|
),
|
|
# Test ENTITY_FIELD keywords as column names
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::description>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"description",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::owner>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"owner",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::tags>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"tags",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::name>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"name",
|
|
],
|
|
),
|
|
EntityLinkTest(
|
|
"<#E::table::bigquery_gcp.shopify.raw_product_catalog::columns::tests>",
|
|
[
|
|
"table",
|
|
"bigquery_gcp.shopify.raw_product_catalog",
|
|
"columns",
|
|
"tests",
|
|
],
|
|
),
|
|
]
|
|
for x in xs:
|
|
x.validate(entity_link.split(x.entitylink), x.split_list)
|
|
|
|
def test_get_entity_link(self):
|
|
"""We can get entity link for different entities"""
|
|
|
|
table_link = get_entity_link(Table, fqn="service.db.schema.table")
|
|
self.assertEqual(table_link, "<#E::table::service.db.schema.table>")
|
|
|
|
dashboard_link = get_entity_link(Dashboard, fqn="service.dashboard")
|
|
self.assertEqual(dashboard_link, "<#E::dashboard::service.dashboard>")
|
|
|
|
column_link = get_entity_link(Table, fqn="service.db.schema.table", column_name="col")
|
|
self.assertEqual(column_link, "<#E::table::service.db.schema.table::columns::col>")
|