mirror of
https://github.com/open-metadata/OpenMetadata
synced 2026-05-24 09:39:11 +00:00
5009059441
* chore(ingestion): replace black/isort/pycln with ruff - Swap formatter + import-sorter + unused-import tooling for ruff (line-length 120, target py3.10) in ingestion + openmetadata-airflow-apis - Drop dead [tool.mypy] config; basedpyright is the active type checker - Bump requires-python to >=3.10 to match noxfile and CLAUDE.md (3.9 is documented as broken on Mac in noxfile.py) - Bump pre-commit-hooks v2.3 -> v5.0; the new check-json catches four pre-existing JSON issues now excluded with an inline TODO - Update Makefile py_format / py_format_check targets to call ruff * chore(ingestion): grandfather ruff lint violations and apply ruff format - 253 noqa markers added via 'ruff check --add-noqa' across 128 files, freezing existing violations so this PR is a tooling-only swap. Per-rule cleanup tracked in the TODO comment in ingestion/pyproject.toml. - Bulk reformat from black 22.3 -> ruff format @ line-length 120. Cosmetic only: imports balanced (-32/+32), structural keywords balanced (-2221/+2221), no logic changes. - Star-import rules (F403/F405) globally ignored; refactoring wildcard imports across connectors is a separate effort. * chore(ingestion): fix pylint findings surfaced after ruff format - filters.py: drop redundant parens around re.match(...) in `if` (C0325 superfluous-parens) — exposed when ruff format unwrapped them - nosql_adaptor.py: move `# pylint: disable=unused-argument` from the `column:` line to the `def` line so it covers `table` too (W0613) — scope was line-based, lost when ruff split params onto multiple lines - action1xx.py: replace `arguments-differ` with `signature-differs` in the disable directive (was always wrong code) and drop the now-useless `unused-argument` suppression (I0021) * fix(ingestion): make ruff extend-exclude robust to multi-root invocations CI's `make py_format_check` runs from the repo root and passes both `ingestion/` and `./openmetadata-airflow-apis/` to ruff in a single invocation. With multiple root paths, ruff's parallel file discovery races on extend-exclude matching against the project root, so files under `ingestion/src/metadata/generated/` were intermittently scanned and produced ~830 I001 violations. 20-run repro: 10/20 fail without the fix, 20/20 pass with the fix. Each excluded directory now appears twice in extend-exclude: - the project-root-relative pattern (cwd = ingestion/) - the prefixed pattern (cwd = repo root, multi-root invocation) * chore(ingestion): address gitar-bot findings + cross-version pylint disable - openmetadata-airflow-apis/pyproject.toml: switch coverage to module-name source + [tool.coverage.paths] glob remap (matches the ingestion pattern). Drops the hardcoded `env/lib/python3.9/site-packages/...` source path, which broke after the requires-python bump to 3.10. (Finding 1) - ingestion/setup.py: remove dead python_version<'3.9' / >='3.9' guards on mysql-connector-python and testcontainers; promote locust to a regular test dep (was conditionally added under sys.version_info >= (3, 9)). Also remove the now-unused `import sys`. (Finding 3) - ingestion/src/metadata/great_expectations/action1xx.py: cover both arguments-differ (great_expectations 0.18.x parent) and signature-differs (great_expectations 1.x parent) in the pylint disable comment, since CI installs 0.18.x and local often has 1.x. unused-argument covers the unused action_context. The opposite rule fires as I0021 useless-suppression on each environment, which is informational and does not affect pylint's exit code.
117 lines
4.6 KiB
Python
117 lines
4.6 KiB
Python
# Copyright 2025 Collate
|
|
# Licensed under the Collate Community License, Version 1.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""
|
|
Test Credentials helper module
|
|
"""
|
|
|
|
from unittest import TestCase
|
|
|
|
from pydantic import AnyUrl, SecretStr
|
|
|
|
from metadata.generated.schema.security.credentials.gcpCredentials import GCPCredentials
|
|
from metadata.generated.schema.security.credentials.gcpExternalAccount import (
|
|
GcpExternalAccount,
|
|
)
|
|
from metadata.generated.schema.security.credentials.gcpValues import (
|
|
GcpCredentialsValues,
|
|
)
|
|
from metadata.utils.credentials import (
|
|
InvalidPrivateKeyException,
|
|
build_google_credentials_dict,
|
|
set_google_credentials,
|
|
)
|
|
from metadata.utils.logger import Loggers
|
|
|
|
|
|
class TestCredentials(TestCase):
|
|
"""
|
|
Validate credentials handling
|
|
"""
|
|
|
|
def test_build_service_account_google_credentials_dict(self):
|
|
"""
|
|
Check how we can validate GCS values
|
|
"""
|
|
|
|
# Key mocked online
|
|
private_key = """-----BEGIN RSA PRIVATE KEY-----
|
|
MIICXQIBAAKBgQDMGwM93kIt3D4r4+dWAGdoTboSaZcFLhsG1lvnZlYEpnZoFo1M
|
|
ek7laRKDUW3CkdTlSid9p4/RTs9SYKuuXvNKNSLApHUeR2zgKBIHYTGGv1t1bEWc
|
|
ohVeqr7w8HkFr9LV4qxgFEWBBd3QYncY/Y1iZgTtbmMiUxJN9vj/kuH0xQIDAQAB
|
|
AoGAPDqAY2JRrwy9v9/ZpPQrj4jYLpS//sRTL1pT9l2pZmfkquR0v6ub2nB+CQgf
|
|
VnoIE70lGBw5AS+7V/i00JiuO6GP/MWWqxKdc5McjBGYDIb+9gQ/DrryVDHsqgGX
|
|
iZrWr7rIrpGsbCB2xt2HPpKR7D9IpI8FA+EEU9fIPfETM6ECQQDv69L78zdijSNk
|
|
CYx70dVHqCiDZT5RbkJqDmQwKabIGXBqZLTM+7ZAHotq0EXGc5BvQGyIMso/qIOs
|
|
Wq3imi3dAkEA2ci4xEzj5guQcGxoVcxfGm+M/VqXLuw/eW1sYdOp52OwdDywxG+I
|
|
6tpm5ByVowhqT8PHDJVOy8GEV9QNw0Y4CQJBAJiyn/rJJlPr/j1aMnZP642KwhY2
|
|
pr4PDegQNsXMjKDISBr+82+POMSAbD1UR0RyItgbybe5k62GZB+bKxaRCGUCQEVj
|
|
l8MrwH0eeCHp2IBlwnN40VIz1/GiYkL9I0g0GXFZKPKQF74uz1AM0DWkCeVNHBpY
|
|
BYaz18xB1znonY33RIkCQQDE3wAWxFrvr582J12qJkE4enmNhRJFdcSREDX54d/5
|
|
VEhPQF0i0tUU7Fl071hcYaiQoZx4nIjN+NG6p5QKbl6k
|
|
-----END RSA PRIVATE KEY-----"""
|
|
|
|
gcp_values = GcpCredentialsValues(
|
|
type="service_account",
|
|
projectId=["project_id"],
|
|
privateKeyId="private_key_id",
|
|
privateKey=private_key,
|
|
clientEmail="email@mail.com",
|
|
clientId="client_id",
|
|
clientX509CertUrl=AnyUrl("http://localhost:1234"),
|
|
)
|
|
|
|
expected_dict = {
|
|
"type": "service_account",
|
|
"project_id": ["project_id"],
|
|
"private_key_id": "private_key_id",
|
|
"private_key": private_key,
|
|
"client_email": "email@mail.com",
|
|
"client_id": "client_id",
|
|
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
|
|
"token_uri": "https://oauth2.googleapis.com/token",
|
|
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
|
|
"client_x509_cert_url": "http://localhost:1234/",
|
|
}
|
|
|
|
self.assertEqual(expected_dict, build_google_credentials_dict(gcp_values))
|
|
|
|
gcp_values.privateKey = SecretStr("I don't think I am a proper Private Key")
|
|
|
|
with self.assertRaises(InvalidPrivateKeyException):
|
|
build_google_credentials_dict(gcp_values)
|
|
|
|
def test_build_external_account_google_credentials_dict(self):
|
|
"""
|
|
Check how we can validate GCS values
|
|
"""
|
|
gcp_values = GcpExternalAccount(
|
|
externalType="external_account",
|
|
audience="audience",
|
|
subjectTokenType="subject_token_type",
|
|
tokenURL="token_url",
|
|
credentialSource={"environmentId": "environment_id"},
|
|
)
|
|
|
|
expected_dict = {
|
|
"type": "external_account",
|
|
"audience": "audience",
|
|
"subject_token_type": "subject_token_type",
|
|
"token_url": "token_url",
|
|
"credential_source": {"environmentId": "environment_id"},
|
|
}
|
|
|
|
self.assertEqual(expected_dict, build_google_credentials_dict(gcp_values))
|
|
with self.assertLogs(Loggers.UTILS.value, level="INFO") as log:
|
|
set_google_credentials(GCPCredentials(gcpConfig=gcp_values, gcpImpersonateServiceAccount=None))
|
|
self.assertIn(
|
|
"Using External account credentials to authenticate with GCP services.",
|
|
log.output[0],
|
|
)
|