OpenMetadata/.github/workflows/playwright-knowledge-graph-postgresql-e2e.yml

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

name: Postgresql PR Knowledge Graph E2E Tests
on:
  workflow_dispatch:
  pull_request:
    types:
      - opened
      - synchronize
      - reopened
      - ready_for_review
    paths:
      - ".github/actions/setup-openmetadata-test-environment/action.yml"
      - ".github/workflows/playwright-knowledge-graph-postgresql-e2e.yml"
      - "docker/run_local_docker.sh"
      - "docker/run_local_docker_common.sh"
      - "docker/run_local_docker_rdf.sh"
      - "docker/validate_compose.py"
      - "docker/development/docker-compose-fuseki.yml"
      - "docker/development/docker-compose-postgres-fuseki.yml"
      - "docs/rdf-local-development.md"
      - "openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/**"
      - "openmetadata-service/src/main/java/org/openmetadata/service/rdf/**"
      - "openmetadata-service/src/main/java/org/openmetadata/service/resources/rdf/**"
      - "openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/**"
      - "openmetadata-service/src/test/java/org/openmetadata/service/rdf/**"
      - "openmetadata-service/src/test/java/org/openmetadata/service/resources/rdf/**"
      - "openmetadata-spec/src/main/resources/rdf/**"
      - "openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeGraph.spec.ts"
      - "openmetadata-ui/src/main/resources/ui/playwright.config.ts"
      - "openmetadata-ui/src/main/resources/ui/src/components/KnowledgeGraph/**"
      - "openmetadata-ui/src/main/resources/ui/src/components/OntologyExplorer/**"
      - "openmetadata-ui/src/main/resources/ui/src/rest/rdfAPI.ts"
      - "openmetadata-ui/src/main/resources/ui/src/types/knowledgeGraph.types.ts"
      - "openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx"

permissions:
  contents: read

concurrency:
  group: playwright-knowledge-graph-pr-postgresql-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

jobs:
  build:
    runs-on: ubuntu-latest
    if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}

      - name: Setup JDK 21
        uses: actions/setup-java@v4
        with:
          java-version: '21'
          distribution: 'temurin'

      - name: Cache Maven Dependencies
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-            

      - name: Install antlr cli
        run: sudo make install_antlr_cli

      - name: Build with Maven
        run: mvn -DskipTests clean package

      - name: Upload Maven build artifact
        uses: actions/upload-artifact@v4
        with:
          name: openmetadata-build
          path: openmetadata-dist/target/openmetadata-*.tar.gz
          retention-days: 1

  playwright-knowledge-graph-postgresql:
    needs: [build]
    runs-on: ubuntu-latest
    if: ${{ !cancelled() && needs.build.result == 'success' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }}
    environment: test
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          swap-storage: true
          docker-images: false

      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}

      - name: Prepare temporary directory for Maven build artifact
        run: mkdir -p "${{ runner.temp }}/openmetadata-build-artifact"

      - name: Download Maven build artifact
        uses: actions/download-artifact@v4
        with:
          name: openmetadata-build
          path: ${{ runner.temp }}/openmetadata-build-artifact

      - name: Copy Maven build artifact into workspace
        run: |
          mkdir -p openmetadata-dist/target
          cp -a "${{ runner.temp }}/openmetadata-build-artifact/." openmetadata-dist/target/          

      - name: Setup Openmetadata Test Environment
        uses: ./.github/actions/setup-openmetadata-test-environment
        with:
          python-version: "3.10"
          args: "-d postgresql -s true"
          startup-script: "./docker/run_local_docker_rdf.sh"
          ingestion_dependency: "all"

      - name: Wait for Fuseki to be healthy
        run: |
          echo "Verifying Fuseki is healthy before running tests..."
          for i in $(seq 1 30); do
            if curl -sf "http://localhost:3030/\$/ping" > /dev/null 2>&1; then
              echo "Fuseki is healthy"
              exit 0
            fi
            echo "Waiting for Fuseki ($i/30)..."
            sleep 10
          done
          echo "Fuseki failed health check. Container logs:"
          docker logs openmetadata-fuseki --tail 100
          exit 1          

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"

      - name: Install dependencies
        working-directory: openmetadata-ui/src/main/resources/ui/
        run: yarn --ignore-scripts --frozen-lockfile

      - name: Install Playwright Browsers
        run: npx playwright@1.57.0 install chromium --with-deps

      - name: Run Knowledge Graph Playwright tests
        working-directory: openmetadata-ui/src/main/resources/ui/
        run: npx playwright test --project="Knowledge Graph"
        env:
          PLAYWRIGHT_IS_OSS: true
          PLAYWRIGHT_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
          PLAYWRIGHT_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD }}
          PLAYWRIGHT_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
          PLAYWRIGHT_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE }}
          PLAYWRIGHT_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
          PLAYWRIGHT_SNOWFLAKE_PASSPHRASE: ${{ secrets.TEST_SNOWFLAKE_PASSPHRASE }}
          PLAYWRIGHT_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY }}
          PLAYWRIGHT_BQ_PROJECT_ID: ${{ secrets.PLAYWRIGHT_BQ_PROJECT_ID }}
          PLAYWRIGHT_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
          PLAYWRIGHT_BQ_PROJECT_ID_TAXONOMY: ${{ secrets.TEST_BQ_PROJECT_ID_TAXONOMY }}
          PLAYWRIGHT_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
          PLAYWRIGHT_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
          PLAYWRIGHT_REDSHIFT_HOST: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
          PLAYWRIGHT_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
          PLAYWRIGHT_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
          PLAYWRIGHT_REDSHIFT_DATABASE: ${{ secrets.TEST_REDSHIFT_DATABASE }}
          PLAYWRIGHT_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
          PLAYWRIGHT_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
          PLAYWRIGHT_METABASE_DB_SERVICE_NAME: ${{ secrets.TEST_METABASE_DB_SERVICE_NAME }}
          PLAYWRIGHT_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
          PLAYWRIGHT_SUPERSET_USERNAME: ${{ secrets.TEST_SUPERSET_USERNAME }}
          PLAYWRIGHT_SUPERSET_PASSWORD: ${{ secrets.TEST_SUPERSET_PASSWORD }}
          PLAYWRIGHT_SUPERSET_HOST_PORT: ${{ secrets.TEST_SUPERSET_HOST_PORT }}
          PLAYWRIGHT_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.TEST_KAFKA_BOOTSTRAP_SERVERS }}
          PLAYWRIGHT_KAFKA_SCHEMA_REGISTRY_URL: ${{ secrets.TEST_KAFKA_SCHEMA_REGISTRY_URL }}
          PLAYWRIGHT_GLUE_ACCESS_KEY: ${{ secrets.TEST_GLUE_ACCESS_KEY }}
          PLAYWRIGHT_GLUE_SECRET_KEY: ${{ secrets.TEST_GLUE_SECRET_KEY }}
          PLAYWRIGHT_GLUE_AWS_REGION: ${{ secrets.TEST_GLUE_AWS_REGION }}
          PLAYWRIGHT_GLUE_ENDPOINT: ${{ secrets.TEST_GLUE_ENDPOINT }}
          PLAYWRIGHT_GLUE_STORAGE_SERVICE: ${{ secrets.TEST_GLUE_STORAGE_SERVICE }}
          PLAYWRIGHT_MYSQL_USERNAME: ${{ secrets.TEST_MYSQL_USERNAME }}
          PLAYWRIGHT_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
          PLAYWRIGHT_MYSQL_HOST_PORT: ${{ secrets.TEST_MYSQL_HOST_PORT }}
          PLAYWRIGHT_MYSQL_DATABASE_SCHEMA: ${{ secrets.TEST_MYSQL_DATABASE_SCHEMA }}
          PLAYWRIGHT_POSTGRES_USERNAME: ${{ secrets.TEST_POSTGRES_USERNAME }}
          PLAYWRIGHT_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
          PLAYWRIGHT_POSTGRES_HOST_PORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
          PLAYWRIGHT_POSTGRES_DATABASE: ${{ secrets.TEST_POSTGRES_DATABASE }}
          PLAYWRIGHT_AIRFLOW_HOST_PORT: ${{ secrets.TEST_AIRFLOW_HOST_PORT }}
          PLAYWRIGHT_ML_MODEL_TRACKING_URI: ${{ secrets.TEST_ML_MODEL_TRACKING_URI }}
          PLAYWRIGHT_ML_MODEL_REGISTRY_URI: ${{ secrets.TEST_ML_MODEL_REGISTRY_URI }}
          PLAYWRIGHT_S3_STORAGE_ACCESS_KEY_ID: ${{ secrets.TEST_S3_STORAGE_ACCESS_KEY_ID }}
          PLAYWRIGHT_S3_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.TEST_S3_STORAGE_SECRET_ACCESS_KEY }}
          PLAYWRIGHT_S3_STORAGE_END_POINT_URL: ${{ secrets.TEST_S3_STORAGE_END_POINT_URL }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - uses: actions/upload-artifact@v4
        if: ${{ !cancelled() }}
        with:
          name: playwright-knowledge-graph-report
          path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
          retention-days: 5

      - name: Clean Up
        if: always()
        run: |
          docker compose -f docker/development/docker-compose-postgres.yml -f docker/development/docker-compose-fuseki.yml down --remove-orphans || true
          docker compose -f docker/development/docker-compose-postgres.yml down --remove-orphans || true
          sudo rm -rf ${PWD}/docker/development/docker-volume