OpenMetadata/.github/workflows/security-scan.yml

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

name: security-scan
on:
  schedule:
    - cron: "0 0 */2 * *"
  workflow_dispatch:

jobs:
  vulnerability-scan:
    runs-on: ubuntu-latest
    environment: security-scan
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"

      - name: Enable yarn
        run: corepack enable

      - name: Install UI dependencies
        working-directory: openmetadata-ui/src/main/resources/ui
        run: yarn install --frozen-lockfile --ignore-scripts

      - name: Run Retire.js scan
        id: retire-scan
        continue-on-error: true
        working-directory: openmetadata-ui/src/main/resources/ui
        run: |
          npx retire@5 \
            --path node_modules/ \
            --severity high \
            --outputformat json \
            --outputpath retire-report.json          

      - name: Verify report was generated
        working-directory: openmetadata-ui/src/main/resources/ui
        run: |
          if [ ! -f retire-report.json ]; then
            echo '::error::retire-report.json was not generated — retire scan may have crashed'
            exit 1
          fi          

      - name: Upload Retire.js Report
        if: success()
        uses: actions/upload-artifact@v4
        with:
          name: retire-js-report
          path: openmetadata-ui/src/main/resources/ui/retire-report.json
          retention-days: 30

      - name: Publish Retire.js Summary
        if: success()
        working-directory: openmetadata-ui/src/main/resources/ui
        run: |
          python3 - << 'EOF' >> $GITHUB_STEP_SUMMARY
          import json

          SEVERITY_ICON = {"critical": "🚨", "high": "🔴", "medium": "🟠", "low": "🟡"}
          SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3}
          NM = "node_modules/"

          def escape(text):
              return str(text).replace('|', '\\|').replace('`', "'")

          try:
              with open("retire-report.json") as f:
                  data = json.load(f)
          except FileNotFoundError:
              print("## Retire.js Scan Results\n\n> Report file not found — scan may not have run.")
              raise SystemExit(0)

          findings = data.get("data", [])
          libs = {}
          for item in findings:
              filepath = item.get("file", "")
              short = filepath[filepath.find(NM) + len(NM):] if NM in filepath else filepath
              for result in item.get("results", []):
                  key = (result.get("component", ""), result.get("version", ""))
                  if key not in libs:
                      libs[key] = {"files": [], "vulns": result.get("vulnerabilities", [])}
                  if short not in libs[key]["files"]:
                      libs[key]["files"].append(short)

          print("## Retire.js Scan Results\n")

          if not libs:
              print("✅ No vulnerable libraries found.")
          else:
              total_vulns = sum(len(v["vulns"]) for v in libs.values())
              print(f"> **{len(libs)} vulnerable librar{'y' if len(libs) == 1 else 'ies'} · {total_vulns} CVE{'s' if total_vulns != 1 else ''} found**\n")

              for (component, version), info in sorted(libs.items(), key=lambda x: min(
                      (SEVERITY_ORDER.get(v.get("severity", "low"), 3) for v in x[1]["vulns"]), default=3)):
                  top_sev = min(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3))
                  icon = SEVERITY_ICON.get(top_sev.get("severity", "low"), "⚪")
                  print(f"### {icon} {component} {version}\n")
                  print("| Severity | CVE | Summary |")
                  print("|---|---|---|")
                  for vuln in sorted(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3)):
                      sev = vuln.get("severity", "")
                      ids = vuln.get("identifiers", {})
                      cves = ids.get("CVE", [])
                      summary = ids.get("summary", "").split("\n")[0][:120]
                      cve_str = ", ".join(f"[{c}](https://nvd.nist.gov/vuln/detail/{c})" for c in cves) if cves else ids.get("githubID", "—")
                      print(f"| {SEVERITY_ICON.get(sev, '')} {sev} | {escape(cve_str)} | {escape(summary)} |")
                  print("\n**Bundled in:**")
                  for f in info["files"]:
                      print(f"- `{f}`")
                  print()
          EOF          

      - name: Force failure on vulnerabilities found
        if: steps.retire-scan.outcome == 'failure'
        run: exit 1

  security-scan:
    runs-on: ubuntu-latest
    environment: security-scan
    env:
      SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
      SNYK_ORGANIZATION: ${{ secrets.SNYK_ORGANIZATION_ID }}

    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          tool-cache: false
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          docker-images: true
          swap-storage: true
      - uses: actions/checkout@v4

      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Set up JDK 21
        uses: actions/setup-java@v4
        with:
          java-version: "21"
          distribution: "temurin"

      - name: Install Ubuntu dependencies
        run: |
          # stop relying on apt cache of GitHub runners
          sudo apt-get update
          sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
          librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf libkrb5-dev          

      # Install and Authenticate to Snyk
      - name: Install Snyk & Authenticate
        run: |
          sudo make install_antlr_cli
          sudo npm install -g snyk
          snyk auth ${SNYK_TOKEN}
          snyk config set org=${SNYK_ORGANIZATION}          

      - name: Install Python dependencies
        run: |
          python3 -m venv env
          source env/bin/activate
          make install_all install_apis          

      - name: Maven build
        id: maven-build
        continue-on-error: true
        run: mvn -DskipTests clean install

      - name: Run Scan
        id: security-report
        if: steps.maven-build.outcome == 'success'
        continue-on-error: true
        run: |
          source env/bin/activate
          rm -rf security-report
          mkdir -p security-report
          # Run snyk subtargets directly; skip `export-snyk-pdf-report` which deletes JSONs after PDF conversion.
          make snyk-ingestion-report || true
          make snyk-ingestion-base-slim-report || true
          make snyk-airflow-apis-report || true
          make snyk-server-report || true
          make snyk-ui-report || true          

      - name: Publish Snyk Summary
        id: snyk-summary
        if: always() && steps.maven-build.outcome == 'success'
        run: |
          python3 scripts/snyk_summary.py security-report \
            --counts-file security-report/_counts.json \
            --slack-file security-report/_slack.txt \
            >> $GITHUB_STEP_SUMMARY
          # Expose counts as step output for downstream gating.
          counts=$(cat security-report/_counts.json)
          echo "counts=$counts" >> $GITHUB_OUTPUT
          high=$(jq '.high + .critical' security-report/_counts.json)
          echo "high_critical=$high" >> $GITHUB_OUTPUT          

      - name: Fail on high/critical Snyk findings
        if: always() && steps.snyk-summary.outputs.high_critical != '' && steps.snyk-summary.outputs.high_critical != '0'
        run: |
          echo "::error::Snyk found ${{ steps.snyk-summary.outputs.high_critical }} high/critical vulnerabilities (see Job Summary)"
          exit 1          

      - name: Generate Snyk HTML/PDF
        if: always() && steps.maven-build.outcome == 'success'
        run: |
          # Back up JSONs because html_to_pdf.py deletes them after PDF conversion.
          mkdir -p /tmp/snyk-json-backup
          cp security-report/*.json /tmp/snyk-json-backup/ 2>/dev/null || true
          make export-snyk-pdf-report || true
          # Restore JSONs alongside generated PDFs/HTMLs.
          cp /tmp/snyk-json-backup/*.json security-report/ 2>/dev/null || true          

      - name: Upload Snyk Reports
        if: always() && steps.maven-build.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: security-report
          path: security-report
          retention-days: 30

      - name: Force failure
        if: steps.maven-build.outcome != 'success' || steps.security-report.outcome != 'success'
        run: |
          exit 1          

  notify:
    runs-on: ubuntu-latest
    environment: security-scan
    needs: [vulnerability-scan, security-scan]
    if: always()
    steps:
      - name: Download Snyk artifact
        if: needs.security-scan.result != 'skipped'
        uses: actions/download-artifact@v4
        with:
          name: security-report
          path: security-report
        continue-on-error: true

      - name: Build Slack payload
        id: build
        run: |
          retire="${{ needs.vulnerability-scan.result }}"
          snyk="${{ needs.security-scan.result }}"
          status_icon() {
            case "$1" in
              success)   echo "✅" ;;
              cancelled) echo "⚠️ (cancelled)" ;;
              skipped)   echo "⚠️ (skipped)" ;;
              *)         echo "❌" ;;
            esac
          }
          retire_icon=$(status_icon "$retire")
          snyk_icon=$(status_icon "$snyk")
          if [ "$retire" = "success" ] && [ "$snyk" = "success" ]; then
            icon="🟢"
          elif [ "$retire" = "failure" ] || [ "$snyk" = "failure" ]; then
            icon="🚨"
          else
            icon="⚠️"
          fi
          run_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          {
            echo "$icon *Security scan* — *OpenMetadata Repo* on branch \`${{ github.ref_name }}\`"
            echo "• Vulnerability scan (Retire.js): $retire_icon"
            echo "• Security scan (Snyk): $snyk_icon"
            echo "<$run_url|Open run details>"
            if [ -f security-report/_slack.txt ]; then
              echo
              cat security-report/_slack.txt
            fi
          } > slack_body.txt
          jq -Rs '{text: ., mrkdwn: true}' slack_body.txt > payload.json          

      - name: Send Slack Notification
        uses: slackapi/slack-github-action@v1.27.1
        with:
          channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
          payload-file-path: payload.json
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}