From df729c02248a4760d50c3333e03977e5371e5cf0 Mon Sep 17 00:00:00 2001 From: Adish M Date: Tue, 11 Jun 2024 12:00:38 +0530 Subject: [PATCH] Run CI fix to main --- .github/workflows/ci.yml | 301 +++++++++++++++++++---- .github/workflows/update-test-system.yml | 6 +- 2 files changed, 258 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc4c36825e..f27e6c04eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,15 +4,11 @@ on: push: branches: [develop, main] pull_request: - types: [labeled, opened, synchronize, reopened] + types: [labeled, unlabeled, closed] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main') && github.run_number || github.ref }} - cancel-in-progress: true - env: FORCE_COLOR: true NODE_OPTIONS: "--max-old-space-size=4096" @@ -25,75 +21,287 @@ env: PG_PASS: postgres PG_DB: tooljet_test -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: build: runs-on: ubuntu-latest - if: | - contains(github.event.pull_request.labels.*.name, 'run-ci') || - github.ref == 'refs/heads/main' || - github.ref == 'refs/heads/develop' + if: ${{ github.event.action == 'labeled' && github.event.label.name == 'run-ci' }} steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v3 - name: Use Node.js 18.18.2 uses: actions/setup-node@v3 with: node-version: 18.18.2 - - name: Cache node modules + # Cache server node modules to speed up subsequent builds + - name: Cache server node modules uses: actions/cache@v3 - env: - cache-name: cache-node-modules with: - # npm cache files are stored in `~/.npm` on Linux/macOS - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + path: server/node_modules + key: ${{ runner.os }}-node-server-${{ hashFiles('server/package-lock.json') }} restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- + ${{ runner.os }}-node-server- - - run: npm run build + # Cache frontend node modules to speed up subsequent builds + - name: Cache frontend node modules + uses: actions/cache@v3 + with: + path: frontend/node_modules + key: ${{ runner.os }}-node-frontend-${{ hashFiles('frontend/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-frontend- - lint: + # Cache plugins node modules to speed up subsequent builds + - name: Cache plugins node modules + uses: actions/cache@v3 + with: + path: plugins/node_modules + key: ${{ runner.os }}-node-plugins-${{ hashFiles('plugins/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-plugins- + + - name: Setup Python 3 + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools + + - name: Install Node.js dependencies + run: npm ci + + - name: Build + run: npm run build + + - name: Frontend ci + run: | + npm --prefix frontend ci + + - name: Server ci + run: npm --prefix server ci + + - name: plugins ci + run: npm --prefix plugins ci + + # Upload plugins build artifacts + - name: Archive specific plugins files and folders + uses: actions/upload-artifact@v4 + with: + name: plugins-files + path: | + plugins/dist + plugins/client.js + plugins/node_modules + plugins/packages/common + plugins/package.json + + # Upload server build artifacts + - name: Archive specific server files and folders + uses: actions/upload-artifact@v4 + with: + name: server-files + path: | + server/dist + + # Upload frontend build artifacts + - name: Archive specific frontend files and folders + uses: actions/upload-artifact@v4 + with: + name: frontend-files + path: | + frontend/build + + lint-for-plugins: runs-on: ubuntu-latest - if: | - contains(github.event.pull_request.labels.*.name, 'run-ci') || - github.ref == 'refs/heads/main' || - github.ref == 'refs/heads/develop' + needs: build + steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v3 - name: Use Node.js 18.18.2 uses: actions/setup-node@v3 with: node-version: 18.18.2 - - name: Cache node modules + # Cache server node modules to speed up subsequent builds + - name: Cache server node modules uses: actions/cache@v3 - env: - cache-name: cache-node-modules with: - # npm cache files are stored in `~/.npm` on Linux/macOS - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + path: server/node_modules + key: ${{ runner.os }}-node-server-${{ hashFiles('server/package-lock.json') }} restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- + ${{ runner.os }}-node-server- - - run: npm run build:plugins - - run: npm --prefix frontend ci && npm --prefix server ci && npm --prefix plugins ci - - run: npm --prefix server run lint && npm --prefix frontend run lint && npm --prefix plugins run lint + # Cache frontend node modules to speed up subsequent builds + - name: Cache frontend node modules + uses: actions/cache@v3 + with: + path: frontend/node_modules + key: ${{ runner.os }}-node-frontend-${{ hashFiles('frontend/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-frontend- + + # Cache plugins node modules to speed up subsequent builds + - name: Cache plugins node modules + uses: actions/cache@v3 + with: + path: plugins/node_modules + key: ${{ runner.os }}-node-plugins-${{ hashFiles('plugins/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-plugins- + + - name: Setup Python 3 + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools + + # Download plugins build artifacts + - name: Download plugins files and folders + uses: actions/download-artifact@v4 + with: + name: plugins-files + + - name: Running for plugins + run: | + npm --prefix plugins run lint + + + lint-for-frontend: + runs-on: ubuntu-latest + needs: build + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Use Node.js 18.18.2 + uses: actions/setup-node@v3 + with: + node-version: 18.18.2 + + # Cache server node modules to speed up subsequent builds + - name: Cache server node modules + uses: actions/cache@v3 + with: + path: server/node_modules + key: ${{ runner.os }}-node-server-${{ hashFiles('server/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-server- + + # Cache frontend node modules to speed up subsequent builds + - name: Cache frontend node modules + uses: actions/cache@v3 + with: + path: frontend/node_modules + key: ${{ runner.os }}-node-frontend-${{ hashFiles('frontend/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-frontend- + + # Cache plugins node modules to speed up subsequent builds + - name: Cache plugins node modules + uses: actions/cache@v3 + with: + path: plugins/node_modules + key: ${{ runner.os }}-node-plugins-${{ hashFiles('plugins/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-plugins- + + - name: Setup Python 3 + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools + + # Download frontend build artifacts + - name: Download frontend files and folders + uses: actions/download-artifact@v4 + with: + name: frontend-files + + - name: Running for frontend + run: | + npm --prefix frontend run lint + + lint-for-server: + runs-on: ubuntu-latest + needs: build + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Use Node.js 18.18.2 + uses: actions/setup-node@v3 + with: + node-version: 18.18.2 + + # Cache server node modules to speed up subsequent builds + - name: Cache server node modules + uses: actions/cache@v3 + with: + path: server/node_modules + key: ${{ runner.os }}-node-server-${{ hashFiles('server/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-server- + + # Cache frontend node modules to speed up subsequent builds + - name: Cache frontend node modules + uses: actions/cache@v3 + with: + path: frontend/node_modules + key: ${{ runner.os }}-node-frontend-${{ hashFiles('frontend/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-frontend- + + # Cache plugins node modules to speed up subsequent builds + - name: Cache plugins node modules + uses: actions/cache@v3 + with: + path: plugins/node_modules + key: ${{ runner.os }}-node-plugins-${{ hashFiles('plugins/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-plugins- + + - name: Setup Python 3 + uses: actions/setup-python@v2 + with: + python-version: 3.x + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools + + # Download server build artifacts + - name: Download server files and folders + uses: actions/download-artifact@v4 + with: + name: server-files + + - name: Running for server + run: | + npm --prefix server run lint unit-test: runs-on: ubuntu-latest + timeout-minutes: 30 # Set a timeout of 30 minutes needs: build - container: node:18.18.2-buster + container: node:18.18.2-bullseye services: postgres: image: postgres @@ -119,7 +327,7 @@ jobs: ${{ runner.os }}-build-${{ env.cache-name }}- ${{ runner.os }}-build- ${{ runner.os }}- - - run: apt update && apt install -y postgresql + - run: apt update && apt install -y postgresql-client - run: npm --prefix plugins ci - run: npm --prefix plugins run create:client && npm --prefix plugins run create:server - run: npm --prefix plugins run build:packages && npm --prefix plugins run build:server @@ -130,8 +338,9 @@ jobs: e2e-test: runs-on: ubuntu-latest + timeout-minutes: 30 # Set a timeout of 30 minutes needs: build - container: node:18.18.2-buster + container: node:18.18.2-bullseye services: postgres: image: postgres @@ -157,11 +366,11 @@ jobs: ${{ runner.os }}-build-${{ env.cache-name }}- ${{ runner.os }}-build- ${{ runner.os }}- - - run: apt update && apt install -y postgresql + - run: apt update && apt install -y postgresql-client - run: npm --prefix plugins ci - run: npm --prefix plugins run create:client && npm --prefix plugins run create:server - run: npm --prefix plugins run build:packages && npm --prefix plugins run build:server - run: npm --prefix server ci - run: npm --prefix server run db:create - run: npm --prefix server run db:migrate - - run: NODE_OPTIONS=--max_old_space_size=8096 npm --prefix server run test:e2e -- --silent --testTimeout=20000 + - run: NODE_OPTIONS=--max_old_space_size=8096 npm --prefix server run test:e2e -- --silent --testTimeout=20000 \ No newline at end of file diff --git a/.github/workflows/update-test-system.yml b/.github/workflows/update-test-system.yml index a33980956b..94bba1b3e9 100644 --- a/.github/workflows/update-test-system.yml +++ b/.github/workflows/update-test-system.yml @@ -97,11 +97,11 @@ jobs: # Remove the existing tooljet/* images sudo docker images -a | grep 'tooljet/' | awk '{print $3}' | xargs sudo docker rmi -f - #checking images + # Check remaining images sudo docker images - # Update docker-compose.yml with the new image - sed -i '/^[[:space:]]*tooljet:/,/^$/ s|^\([[:space:]]*image:[[:space:]]*\).*|\1tooljet/tj-osv:${{ env.SAFE_BRANCH_NAME }}|' docker-compose.yaml + # Update docker-compose.yml with the new image for tooljet service + sed -i '/^[[:space:]]*tooljet:/,/^[[:space:]]*[^:]*$/ { /^[[:space:]]*image:[[:space:]]*tooljet\/tj-osv/s|\(image:[[:space:]]*\).*|\1tooljet/tj-osv:'"${{ env.SAFE_BRANCH_NAME }}"'| }' docker-compose.yml # Start the Docker containers cat docker-compose.yaml