mirror of
https://github.com/open-metadata/OpenMetadata
synced 2026-05-24 09:39:11 +00:00
af31136400
* Add Playwright failure reporting to PR comments When a shard fails, automatically post a PR comment listing failed tests with screenshot/trace availability and artifact download links. Updates existing comments on re-runs to avoid duplicates. Also uploads test-results (screenshots, traces) as separate artifacts for easier debugging. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Improve Playwright failure diagnostics - Add JSON reporter for machine-readable test results - Capture traces on every failure (not just retries) for debugging - Record video on failure to see the full interaction sequence - PR comment now distinguishes genuine failures vs flaky tests - Genuine failures show error messages inline with expandable details - Flaky tests listed separately so they're clearly not blockers - Write failure screenshots to GitHub Step Summary for quick viewing Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * Address review: clean up stale comments and paginate lookup - Delete failure comments when shard passes on re-run - Use paginated comment lookup to handle PRs with 100+ comments Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
391 lines
18 KiB
YAML
391 lines
18 KiB
YAML
# Copyright 2021 Collate
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
# This workflow executes end-to-end (e2e) tests using Playwright with PostgreSQL as the database.
|
||
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
|
||
|
||
name: Postgresql PR Playwright E2E Tests
|
||
on:
|
||
workflow_dispatch:
|
||
pull_request_target:
|
||
types:
|
||
- labeled
|
||
- opened
|
||
- synchronize
|
||
- reopened
|
||
- ready_for_review
|
||
paths-ignore:
|
||
- ".github/**"
|
||
- "openmetadata-dist/**"
|
||
- "docker/**"
|
||
- "!docker/development/docker-compose.yml"
|
||
- "!docker/development/docker-compose-postgres.yml"
|
||
- "openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**"
|
||
- "openmetadata-ui/src/main/resources/ui/playwright/docs/**"
|
||
|
||
permissions:
|
||
contents: read
|
||
pull-requests: write
|
||
|
||
concurrency:
|
||
group: playwright-ci-pr-postgresql-${{ github.head_ref || github.run_id }}
|
||
cancel-in-progress: true
|
||
|
||
jobs:
|
||
playwright-ci-postgresql:
|
||
runs-on: ubuntu-latest
|
||
if: ${{ !github.event.pull_request.draft }}
|
||
environment: test
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
shardIndex: [1, 2, 3, 4, 5, 6]
|
||
shardTotal: [6]
|
||
steps:
|
||
- name: Free Disk Space (Ubuntu)
|
||
uses: jlumbroso/free-disk-space@main
|
||
with:
|
||
tool-cache: false
|
||
android: true
|
||
dotnet: true
|
||
haskell: true
|
||
large-packages: false
|
||
swap-storage: true
|
||
docker-images: false
|
||
- name: Wait for the labeler
|
||
uses: lewagon/wait-on-check-action@v1.3.4
|
||
if: ${{ github.event_name == 'pull_request_target' }}
|
||
with:
|
||
ref: ${{ github.event.pull_request.head.sha }}
|
||
check-name: Team Label
|
||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||
wait-interval: 90
|
||
|
||
- name: Verify PR labels
|
||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||
if: ${{ github.event_name == 'pull_request_target' }}
|
||
with:
|
||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||
valid-labels: "safe to test"
|
||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||
disable-reviews: true # To not auto approve changes
|
||
|
||
- name: Checkout
|
||
uses: actions/checkout@v4
|
||
with:
|
||
ref: ${{ github.event.pull_request.head.sha }}
|
||
|
||
- name: Cache Maven Dependencies
|
||
id: cache-output
|
||
uses: actions/cache@v4
|
||
with:
|
||
path: ~/.m2
|
||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||
restore-keys: |
|
||
${{ runner.os }}-maven-
|
||
|
||
- name: Setup Openmetadata Test Environment
|
||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||
with:
|
||
python-version: "3.10"
|
||
# Pass different args depending on the shardIndex:
|
||
# - shard 2: run with ingestion setup
|
||
# - other shards: skip ingestion setup
|
||
args: ${{ matrix.shardIndex == 2 && '-d postgresql' || '-d postgresql -i false' }}
|
||
ingestion_dependency: "all"
|
||
|
||
- name: Setup Node.js
|
||
uses: actions/setup-node@v4
|
||
with:
|
||
node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
|
||
|
||
- name: Install dependencies
|
||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||
run: yarn --ignore-scripts --frozen-lockfile
|
||
- name: Install Playwright Browsers
|
||
run: npx playwright@1.57.0 install --with-deps
|
||
- name: Run Playwright tests
|
||
id: run-tests
|
||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||
run: |
|
||
if [ "${{ matrix.shardIndex }}" -eq "1" ]; then
|
||
echo "🔹 Running DataAssetRules-only tests on shard 1"
|
||
|
||
# The testMatch pattern ensures only DataAssetRules*.spec.ts files run
|
||
npx playwright test \
|
||
--project=setup \
|
||
--project=DataAssetRulesEnabled \
|
||
--project=DataAssetRulesDisabled \
|
||
--project=Basic \
|
||
|
||
elif [ "${{ matrix.shardIndex }}" -eq "2" ]; then
|
||
echo "🔹 Running ingestion tests on shard 2"
|
||
|
||
# run ingestion framework tests
|
||
npx playwright test \
|
||
--project=ingestion \
|
||
|
||
else
|
||
# Shards 3-6 handle chromium tests (4 workers total)
|
||
CHROMIUM_SHARD=$(( ${{ matrix.shardIndex }} - 2 ))
|
||
echo "🔹 Running all other tests (excluding DataAssetRules) on chromium shard ${CHROMIUM_SHARD}/4"
|
||
npx playwright test \
|
||
--project=chromium \
|
||
--grep-invert @dataAssetRules \
|
||
--shard=${CHROMIUM_SHARD}/4
|
||
fi
|
||
|
||
env:
|
||
PLAYWRIGHT_IS_OSS: true
|
||
PLAYWRIGHT_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
|
||
PLAYWRIGHT_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD }}
|
||
PLAYWRIGHT_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
|
||
PLAYWRIGHT_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE }}
|
||
PLAYWRIGHT_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
|
||
PLAYWRIGHT_SNOWFLAKE_PASSPHRASE: ${{ secrets.TEST_SNOWFLAKE_PASSPHRASE }}
|
||
PLAYWRIGHT_PROJECT_ID: ${{ steps.cypress-project-id.outputs.CYPRESS_PROJECT_ID }}
|
||
PLAYWRIGHT_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY }}
|
||
PLAYWRIGHT_BQ_PROJECT_ID: ${{ secrets.PLAYWRIGHT_BQ_PROJECT_ID }}
|
||
PLAYWRIGHT_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
|
||
PLAYWRIGHT_BQ_PROJECT_ID_TAXONOMY: ${{ secrets.TEST_BQ_PROJECT_ID_TAXONOMY }}
|
||
PLAYWRIGHT_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
|
||
PLAYWRIGHT_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
|
||
PLAYWRIGHT_REDSHIFT_HOST: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||
PLAYWRIGHT_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||
PLAYWRIGHT_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||
PLAYWRIGHT_REDSHIFT_DATABASE: ${{ secrets.TEST_REDSHIFT_DATABASE }}
|
||
PLAYWRIGHT_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
|
||
PLAYWRIGHT_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
|
||
PLAYWRIGHT_METABASE_DB_SERVICE_NAME: ${{ secrets.TEST_METABASE_DB_SERVICE_NAME }}
|
||
PLAYWRIGHT_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
|
||
PLAYWRIGHT_SUPERSET_USERNAME: ${{ secrets.TEST_SUPERSET_USERNAME }}
|
||
PLAYWRIGHT_SUPERSET_PASSWORD: ${{ secrets.TEST_SUPERSET_PASSWORD }}
|
||
PLAYWRIGHT_SUPERSET_HOST_PORT: ${{ secrets.TEST_SUPERSET_HOST_PORT }}
|
||
PLAYWRIGHT_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.TEST_KAFKA_BOOTSTRAP_SERVERS }}
|
||
PLAYWRIGHT_KAFKA_SCHEMA_REGISTRY_URL: ${{ secrets.TEST_KAFKA_SCHEMA_REGISTRY_URL }}
|
||
PLAYWRIGHT_GLUE_ACCESS_KEY: ${{ secrets.TEST_GLUE_ACCESS_KEY }}
|
||
PLAYWRIGHT_GLUE_SECRET_KEY: ${{ secrets.TEST_GLUE_SECRET_KEY }}
|
||
PLAYWRIGHT_GLUE_AWS_REGION: ${{ secrets.TEST_GLUE_AWS_REGION }}
|
||
PLAYWRIGHT_GLUE_ENDPOINT: ${{ secrets.TEST_GLUE_ENDPOINT }}
|
||
PLAYWRIGHT_GLUE_STORAGE_SERVICE: ${{ secrets.TEST_GLUE_STORAGE_SERVICE }}
|
||
PLAYWRIGHT_MYSQL_USERNAME: ${{ secrets.TEST_MYSQL_USERNAME }}
|
||
PLAYWRIGHT_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
|
||
PLAYWRIGHT_MYSQL_HOST_PORT: ${{ secrets.TEST_MYSQL_HOST_PORT }}
|
||
PLAYWRIGHT_MYSQL_DATABASE_SCHEMA: ${{ secrets.TEST_MYSQL_DATABASE_SCHEMA }}
|
||
PLAYWRIGHT_POSTGRES_USERNAME: ${{ secrets.TEST_POSTGRES_USERNAME }}
|
||
PLAYWRIGHT_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
|
||
PLAYWRIGHT_POSTGRES_HOST_PORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
|
||
PLAYWRIGHT_POSTGRES_DATABASE: ${{ secrets.TEST_POSTGRES_DATABASE }}
|
||
PLAYWRIGHT_AIRFLOW_HOST_PORT: ${{ secrets.TEST_AIRFLOW_HOST_PORT }}
|
||
PLAYWRIGHT_ML_MODEL_TRACKING_URI: ${{ secrets.TEST_ML_MODEL_TRACKING_URI }}
|
||
PLAYWRIGHT_ML_MODEL_REGISTRY_URI: ${{ secrets.TEST_ML_MODEL_REGISTRY_URI }}
|
||
PLAYWRIGHT_S3_STORAGE_ACCESS_KEY_ID: ${{ secrets.TEST_S3_STORAGE_ACCESS_KEY_ID }}
|
||
PLAYWRIGHT_S3_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.TEST_S3_STORAGE_SECRET_ACCESS_KEY }}
|
||
PLAYWRIGHT_S3_STORAGE_END_POINT_URL: ${{ secrets.TEST_S3_STORAGE_END_POINT_URL }}
|
||
|
||
# Recommended: pass the GitHub token lets this action correctly
|
||
# determine the unique run id necessary to re-run the checks
|
||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
|
||
- uses: actions/upload-artifact@v4
|
||
if: ${{ !cancelled() }}
|
||
with:
|
||
name: playwright-report-${{ matrix.shardIndex }}
|
||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||
retention-days: 5
|
||
|
||
- name: Upload test results (screenshots, traces)
|
||
uses: actions/upload-artifact@v4
|
||
if: ${{ !cancelled() }}
|
||
with:
|
||
name: playwright-test-results-${{ matrix.shardIndex }}
|
||
path: openmetadata-ui/src/main/resources/ui/playwright/output/test-results
|
||
retention-days: 5
|
||
|
||
- name: Post failure summary to PR
|
||
if: ${{ !cancelled() && github.event_name == 'pull_request_target' }}
|
||
uses: actions/github-script@v7
|
||
with:
|
||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||
script: |
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
|
||
const shard = '${{ matrix.shardIndex }}';
|
||
const runId = '${{ github.run_id }}';
|
||
const repo = context.repo;
|
||
const prNumber = context.payload.pull_request?.number;
|
||
if (!prNumber) return;
|
||
|
||
const jsonPath = 'openmetadata-ui/src/main/resources/ui/playwright/output/results.json';
|
||
if (!fs.existsSync(jsonPath)) {
|
||
console.log('No results.json found, skipping comment');
|
||
return;
|
||
}
|
||
|
||
const report = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
|
||
const suites = report.suites || [];
|
||
|
||
// Flatten all test results from nested suites
|
||
const allTests = [];
|
||
function collectTests(suite, filePath) {
|
||
const file = suite.file || filePath || '';
|
||
for (const spec of (suite.specs || [])) {
|
||
for (const test of (spec.tests || [])) {
|
||
const results = test.results || [];
|
||
const lastResult = results[results.length - 1] || {};
|
||
const firstResult = results[0] || {};
|
||
allTests.push({
|
||
title: spec.title,
|
||
file: file,
|
||
status: test.status, // expected, unexpected, flaky, skipped
|
||
duration: results.reduce((sum, r) => sum + (r.duration || 0), 0),
|
||
retries: results.length - 1,
|
||
error: lastResult.error?.message || firstResult.error?.message || '',
|
||
attachments: lastResult.attachments || firstResult.attachments || [],
|
||
});
|
||
}
|
||
}
|
||
for (const child of (suite.suites || [])) {
|
||
collectTests(child, file);
|
||
}
|
||
}
|
||
for (const suite of suites) {
|
||
collectTests(suite, '');
|
||
}
|
||
|
||
const genuine = allTests.filter(t => t.status === 'unexpected');
|
||
const flaky = allTests.filter(t => t.status === 'flaky');
|
||
const passed = allTests.filter(t => t.status === 'expected');
|
||
const skipped = allTests.filter(t => t.status === 'skipped');
|
||
|
||
// Helper to find existing bot comment for this shard
|
||
async function findExistingComment() {
|
||
const marker = `Playwright Shard ${shard}`;
|
||
for await (const response of github.paginate.iterator(
|
||
github.rest.issues.listComments, { ...repo, issue_number: prNumber, per_page: 100 }
|
||
)) {
|
||
const found = response.data.find(c =>
|
||
c.user?.login === 'github-actions[bot]' && c.body?.includes(marker)
|
||
);
|
||
if (found) return found;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// If everything passed, delete any stale failure comment and exit
|
||
if (genuine.length === 0 && flaky.length === 0) {
|
||
console.log(`Shard ${shard}: all ${passed.length} tests passed`);
|
||
const stale = await findExistingComment();
|
||
if (stale) {
|
||
await github.rest.issues.deleteComment({ ...repo, comment_id: stale.id });
|
||
console.log(`Deleted stale failure comment for shard ${shard}`);
|
||
}
|
||
return;
|
||
}
|
||
|
||
const artifactUrl = `https://github.com/${repo.owner}/${repo.repo}/actions/runs/${runId}`;
|
||
const lines = [];
|
||
|
||
if (genuine.length > 0) {
|
||
lines.push(`## 🔴 Playwright Shard ${shard} — ${genuine.length} genuine failure(s)${flaky.length > 0 ? `, ${flaky.length} flaky` : ''}`);
|
||
} else {
|
||
lines.push(`## 🟡 Playwright Shard ${shard} — ${flaky.length} flaky test(s) (all passed on retry)`);
|
||
}
|
||
lines.push('');
|
||
lines.push(`✅ ${passed.length} passed · ❌ ${genuine.length} failed · 🟡 ${flaky.length} flaky · ⏭️ ${skipped.length} skipped`);
|
||
lines.push('');
|
||
|
||
if (genuine.length > 0) {
|
||
lines.push('### Genuine Failures (failed on all attempts)');
|
||
lines.push('');
|
||
for (const t of genuine.slice(0, 20)) {
|
||
const shortFile = t.file.replace(/.*playwright\/e2e\//, '');
|
||
const errorSnippet = t.error.split('\n')[0].substring(0, 200);
|
||
lines.push(`<details><summary>❌ <code>${shortFile}</code> › ${t.title}</summary>`);
|
||
lines.push('');
|
||
lines.push('```');
|
||
lines.push(t.error.substring(0, 1000));
|
||
lines.push('```');
|
||
lines.push('</details>');
|
||
lines.push('');
|
||
}
|
||
if (genuine.length > 20) {
|
||
lines.push(`... and ${genuine.length - 20} more failures`);
|
||
lines.push('');
|
||
}
|
||
}
|
||
|
||
if (flaky.length > 0) {
|
||
lines.push('<details><summary>🟡 Flaky tests (passed on retry)</summary>');
|
||
lines.push('');
|
||
for (const t of flaky.slice(0, 20)) {
|
||
const shortFile = t.file.replace(/.*playwright\/e2e\//, '');
|
||
lines.push(`- \`${shortFile}\` › ${t.title} (${t.retries} ${t.retries === 1 ? 'retry' : 'retries'})`);
|
||
}
|
||
if (flaky.length > 20) {
|
||
lines.push(`- ... and ${flaky.length - 20} more`);
|
||
}
|
||
lines.push('');
|
||
lines.push('</details>');
|
||
lines.push('');
|
||
}
|
||
|
||
lines.push(`📦 [Download artifacts (report + screenshots + traces + videos)](${artifactUrl})`);
|
||
lines.push('');
|
||
lines.push('<details><summary>How to debug locally</summary>');
|
||
lines.push('');
|
||
lines.push('```bash');
|
||
lines.push('# Download playwright-test-results-' + shard + ' artifact and unzip');
|
||
lines.push('npx playwright show-trace path/to/trace.zip # view trace');
|
||
lines.push('# Videos are in the same artifact as .webm files');
|
||
lines.push('```');
|
||
lines.push('</details>');
|
||
|
||
const body = lines.join('\n');
|
||
|
||
const existing = await findExistingComment();
|
||
if (existing) {
|
||
await github.rest.issues.updateComment({ ...repo, comment_id: existing.id, body });
|
||
} else {
|
||
await github.rest.issues.createComment({ ...repo, issue_number: prNumber, body });
|
||
}
|
||
|
||
// Write a rich summary with inline screenshots to GITHUB_STEP_SUMMARY
|
||
const summaryLines = [`## Shard ${shard} Failure Screenshots\n`];
|
||
const resultsDir = 'openmetadata-ui/src/main/resources/ui/playwright/output/test-results';
|
||
if (fs.existsSync(resultsDir)) {
|
||
let screenshotCount = 0;
|
||
for (const entry of fs.readdirSync(resultsDir, { withFileTypes: true })) {
|
||
if (!entry.isDirectory()) continue;
|
||
const testDir = path.join(resultsDir, entry.name);
|
||
const pngs = fs.readdirSync(testDir).filter(f => f.endsWith('.png'));
|
||
for (const png of pngs) {
|
||
if (screenshotCount >= 10) break;
|
||
const pngPath = path.join(testDir, png);
|
||
const b64 = fs.readFileSync(pngPath, 'base64');
|
||
const testName = entry.name.replace(/-chromium.*$/, '').replace(/-/g, ' ');
|
||
summaryLines.push(`### ${testName}`);
|
||
summaryLines.push(`<img src="data:image/png;base64,${b64}" width="800" />\n`);
|
||
screenshotCount++;
|
||
}
|
||
if (screenshotCount >= 10) break;
|
||
}
|
||
}
|
||
if (summaryLines.length > 1) {
|
||
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, summaryLines.join('\n'));
|
||
}
|
||
|
||
- name: Clean Up
|
||
run: |
|
||
cd ./docker/development
|
||
docker compose down --remove-orphans
|
||
sudo rm -rf ${PWD}/docker-volume
|