Reduce the size of cache (GH Actions) (#2616)

.github/actions/setup/action.yml

@@ -8,9 +8,6 @@ inputs:
   codegen:
     description: Should run GraphQL Codegen?
     default: 'true'
-  installDependencies:
-    description: Should run pnpm install?
-    default: 'true'
   workingDirectory:
     description: Working dir
     default: ${{ github.workspace }}
@@ -18,15 +15,16 @@ inputs:
     description: Name of the workflow that called this action
     required: true
     type: string
+  cacheNext:
+    description: Should cache Next?
+    default: 'true'
+  cacheTurbo:
+    description: Should cache Turbo?
+    default: 'true'
 
 runs:
   using: 'composite'
   steps:
-    - name: install nodejs
-      uses: actions/setup-node@v3.6.0
-      with:
-        node-version-file: .node-version
-
     - name: check pnpm version
       shell: bash
       id: pnpm
@@ -41,23 +39,15 @@ runs:
       with:
         version: ${{ steps.pnpm.outputs.version }}
 
-    - name: pnpm store path
-      id: pnpm-cache
-      shell: bash
-      run: echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
-
-    - uses: actions/cache@v3
-      name: restore pnpm cache
+    - name: install nodejs
+      uses: actions/setup-node@v3.6.0
       with:
-        path: |
-          ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
-          ~/.cache/Cypress
-        key: ${{ runner.os }}-pnpm-store-n18-${{ hashFiles('**/pnpm-lock.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-pnpm-store-n18
+        node-version-file: .node-version
+        cache: 'pnpm'
 
     - name: Cache Turbo
       uses: actions/cache@v3
+      if: ${{ inputs.cacheNext == 'true' }}
       with:
         path: node_modules/.cache/turbo
         key: turbo-cache-${{ hashFiles('**/pnpm-lock.yaml') }}-${{ inputs.actor }}-${{ github.sha }}
@@ -67,6 +57,7 @@ runs:
 
     - uses: actions/cache@v3
       name: Cache Next.js
+      if: ${{ inputs.cacheNext == 'true' }}
       with:
         path: |
           packages/web/app/.next/cache
@@ -78,7 +69,6 @@ runs:
           nextjs-cache-${{ hashFiles('**/pnpm-lock.yaml') }}-
 
     - name: pnpm install
-      if: ${{ inputs.installDependencies == 'true' }}
       shell: bash
       working-directory: ${{ inputs.workingDirectory }}
       run: pnpm install --frozen-lockfile

.github/workflows/db-types-diff.yaml

@@ -33,6 +33,8 @@ jobs:
         with:
           codegen: false # no need to run codegen in this case, we can skip
           actor: db-types-diff
+          cacheNext: false
+          cacheTurbo: false
 
       - name: create database
         working-directory: packages/migrations

.github/workflows/lint.yaml

@@ -15,6 +15,8 @@ jobs:
         uses: ./.github/actions/setup
         with:
           actor: lint
+          cacheNext: false
+          cacheTurbo: false
 
       - name: lint .env.template files
         run: pnpm lint:env-template

.github/workflows/migrations-test.yaml

@@ -35,6 +35,8 @@ jobs:
         with:
           codegen: false # no need to run codegen in this case, we can skip
           actor: migrations-test
+          cacheNext: false
+          cacheTurbo: false
 
       - name: migrations tests
         run: pnpm test

.github/workflows/pr-closed.yaml

@@ -0,0 +1,40 @@
+name: pr
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  cleanup:
+    name: cleanup
+    runs-on: ubuntu-latest
+    permissions:
+      # `actions:write` permission is required to delete caches
+      #   See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id
+      actions: write
+      contents: read
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Cleanup
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+          BRANCH=refs/pull/${{ github.event.pull_request.number }}/merge
+
+          echo "Fetching list of cache key"
+          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys. 
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeysForPR
+          do
+              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
+          done
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pr-rust.yaml

@@ -9,10 +9,6 @@ on:
       - 'Cargo.toml'
 
 env:
-  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
-  TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
-  TURBO_API_URL: ${{ secrets.TURBO_API_URL }}
-  TURBO_REMOTE_ONLY: 'true'
   HIVE_TOKEN: ${{ secrets.HIVE_TOKEN }}
 
 jobs:

.github/workflows/pr.yaml

@@ -3,10 +3,6 @@ on:
   pull_request: {}
 
 env:
-  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
-  TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
-  TURBO_API_URL: ${{ secrets.TURBO_API_URL }}
-  TURBO_REMOTE_ONLY: 'true'
   HIVE_TOKEN: ${{ secrets.HIVE_TOKEN }}
 
 jobs:

.github/workflows/publish-rust.yaml

@@ -40,10 +40,25 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: check pnpm version
+        shell: bash
+        id: pnpm
+        run: |
+          PNPM_VERSION=$(cat package.json | jq -r '.packageManager' | awk -F@ '{print $2}')
+          PNPM_VERSION=${PNPM_VERSION:-8}
+          echo "Using PNPM version $PNPM_VERSION"
+          echo "version=$PNPM_VERSION" >> $GITHUB_OUTPUT
+
+      - name: install pnpm
+        uses: pnpm/action-setup@v2.2.4
+        with:
+          version: ${{ steps.pnpm.outputs.version }}
+
       - name: Use Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v3.6.0
         with:
           node-version-file: .node-version
+          cache: 'pnpm'
 
       - name: Prepare MacOS
         if: ${{ matrix.os == 'macos-latest' }}

.github/workflows/release-alpha.yaml

@@ -29,6 +29,8 @@ jobs:
         with:
           codegen: false # no need to run because release script will run it anyway
           actor: release-alpha
+          cacheNext: false
+          cacheTurbo: true
 
       - name: build libraries
         run: pnpm build:libraries

.github/workflows/release-stable.yaml

@@ -21,6 +21,8 @@ jobs:
         with:
           codegen: false # no need to run because release script will run it anyway
           actor: release-stable
+          cacheNext: false
+          cacheTurbo: true
 
       - name: prepare npm credentials
         run: |

.github/workflows/storybook.yaml

@@ -28,6 +28,8 @@ jobs:
         with:
           codegen: true
           actor: storybook
+          cacheNext: true
+          cacheTurbo: true
 
       - uses: the-guild-org/shared-config/website-cf@main
         if: steps.storybook-files.outputs.any_changed == 'true'

.github/workflows/tests-e2e.yaml

@@ -29,6 +29,11 @@ jobs:
         with:
           codegen: false
           actor: test-e2e
+          cacheNext: false
+          cacheTurbo: false
+
+      - name: Install Cypress binary
+        run: pnpm cypress install
 
       - name: patch compose file volumes
         uses: mikefarah/yq@v4.34.1

.github/workflows/tests-integration.yaml

@@ -22,6 +22,10 @@ on:
 jobs:
   integration:
     runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        # Divide integration tests into 3 shards, to run them in parallel.
+        shardIndex: [1, 2, 3]
 
     env:
       DOCKER_REGISTRY: ${{ inputs.registry }}/${{ inputs.imageName }}/
@@ -41,6 +45,8 @@ jobs:
         uses: ./.github/actions/setup
         with:
           actor: test-integration
+          cacheNext: true
+          cacheTurbo: true
 
       - name: prepare packages
         run: pnpm --filter integration-tests prepare:env
@@ -66,11 +72,11 @@ jobs:
           docker compose -f docker/docker-compose.community.yml -f ./integration-tests/docker-compose.integration.yaml --env-file ./integration-tests/.env logs
 
       - name: run integration tests
-        timeout-minutes: 15
+        timeout-minutes: 5
         env:
           HIVE_DEBUG: 1
         run: |
-          VITEST_MAX_THREADS=${{ steps.cpu-cores.outputs.count }} pnpm --filter integration-tests test:integration
+          VITEST_MAX_THREADS=${{ steps.cpu-cores.outputs.count }} pnpm --filter integration-tests test:integration --shard=${{ matrix.shardIndex }}/3
 
       - name: log dump
         if: ${{ failure() }}

.github/workflows/tests-unit.yaml

@@ -14,6 +14,8 @@ jobs:
         uses: ./.github/actions/setup
         with:
           actor: test-unit
+          cacheNext: false
+          cacheTurbo: false
 
       - name: unit tests
         run: pnpm test

.github/workflows/typescript-typecheck.yaml

@@ -15,6 +15,8 @@ jobs:
         uses: ./.github/actions/setup
         with:
           actor: typescript-typecheck
+          cacheNext: false
+          cacheTurbo: false
 
       - name: get cpu count
         id: cpu-cores

.github/workflows/website.yaml

@@ -24,6 +24,8 @@ jobs:
         with:
           codegen: false
           actor: website
+          cacheNext: true
+          cacheTurbo: true
 
       - uses: the-guild-org/shared-config/website-cf@main
         if: steps.website-files.outputs.any_changed == 'true'

.gitignore

@@ -71,6 +71,8 @@ typings/
 .nuxt
 dist/
 
+deploy-tmp/
+
 # Gatsby files
 .cache/
 # Comment in the public line in if your project uses Gatsby and *not* Next.js

docker/cli.dockerfile

@@ -6,7 +6,6 @@ WORKDIR /usr/src/app
 COPY . /usr/src/app/
 
 RUN npm install --omit=dev
-RUN npm install -g file:./
 
 LABEL org.opencontainers.image.title=$IMAGE_TITLE
 LABEL org.opencontainers.image.version=$RELEASE
@@ -17,6 +16,6 @@ LABEL org.opencontainers.image.url="https://github.com/kamilkisiela/graphql-hive
 
 ENV ENVIRONMENT production
 ENV RELEASE $RELEASE
-RUN npx hive --version
+RUN npx . --version
 
-ENTRYPOINT ["npx", "hive"]
+ENTRYPOINT ["npx", "."]

package.json

@@ -20,9 +20,9 @@
   },
   "scripts": {
     "build": "pnpm turbo build --color",
-    "build:libraries": "pnpm graphql:generate && pnpm turbo build --filter=./packages/libraries/* --color",
-    "build:services": "pnpm turbo build  --filter=./packages/services/**/* --filter=./packages/migrations --color",
-    "build:web": "pnpm turbo build --filter=./packages/web/* --color",
+    "build:libraries": "pnpm prebuild && pnpm graphql:generate && pnpm turbo build --filter=./packages/libraries/* --color",
+    "build:services": "pnpm prebuild && pnpm turbo build  --filter=./packages/services/**/* --filter=./packages/migrations --color",
+    "build:web": "pnpm prebuild && pnpm turbo build --filter=./packages/web/* --color",
     "cargo:fix": "bash ./scripts/fix-symbolic-link.sh",
     "docker:build": "docker buildx bake -f docker/docker.hcl --load build",
     "env:sync": "tsx scripts/sync-env-files.ts",
@@ -34,8 +34,9 @@
     "lint:fix": "pnpm lint --fix",
     "lint:prettier": "prettier --cache --check .",
     "local:setup": "docker-compose -f ./docker/docker-compose.dev.yml up -d --remove-orphans && pnpm --filter @hive/migrations db:init",
-    "postinstall": "node ./scripts/patch-manifests.js && pnpm env:sync && node ./scripts/turborepo-cleanup.js && node ./scripts/turborepo-setup.js && pnpm cargo:fix",
+    "postinstall": "node ./scripts/patch-manifests.js && pnpm env:sync && node ./scripts/turborepo-cleanup.js && pnpm cargo:fix",
     "pre-commit": "exit 0 && lint-staged",
+    "prebuild": "rimraf deploy-tmp && rimraf packages/**/deploy-tmp",
     "prepare": "husky install",
     "prerelease": "pnpm build:libraries",
     "prettier": "prettier --cache --write --list-different --ignore-unknown \"**/*\"",

packages/web/app/package.json

@@ -8,7 +8,7 @@
     "build-storybook": "storybook build",
     "build:config": "tsup-node --no-splitting --out-dir . --loader \".mts=ts\" --format esm --target node18 next.config.mts",
     "dev": "pnpm build:config && next dev",
-    "postbuild": "pnpm --filter @hive/app deploy --prod --no-optional dist/web && rimraf dist/node_modules && mv dist/web/node_modules dist && rimraf dist/web",
+    "postbuild": "pnpm --filter @hive/app deploy --prod --no-optional deploy-tmp/web && rimraf dist/node_modules && mv deploy-tmp/web/node_modules dist && rimraf deploy-tmp/web",
     "start": "node dist/index.js",
     "storybook": "storybook dev -p 6006",
     "typecheck": "tsc"

scripts/turborepo-setup.js

@@ -1,53 +0,0 @@
-/**
- * Create Turborepo config file
- */
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import dotenv from 'dotenv';
-
-const cwd = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
-
-function loadEnv() {
-  const envFile = path.resolve(cwd, '.env');
-
-  if (!fs.existsSync(envFile)) {
-    return {};
-  }
-
-  return dotenv.parse(fs.readFileSync(envFile, 'utf-8'));
-}
-
-function main() {
-  const configDir = '.turbo';
-  const configFile = path.resolve(cwd, configDir, 'config.json');
-  const envFile = loadEnv();
-
-  const env = {
-    TURBO_API_URL: envFile.TURBO_API_URL || process.env.TURBO_API_URL,
-    TURBO_TOKEN: envFile.TURBO_TOKEN || process.env.TURBO_TOKEN,
-    TURBO_TEAM: envFile.TURBO_TEAM || process.env.TURBO_TEAM,
-  };
-
-  if (!env.TURBO_API_URL || !env.TURBO_TOKEN || !env.TURBO_TEAM) {
-    console.log('[turborepo-setup] No TURBO_API_URL, TURBO_TOKEN or TURBO_TEAM found. Skipping.');
-    return;
-  }
-
-  fs.writeFileSync(
-    configFile,
-    JSON.stringify(
-      {
-        teamId: '_',
-        apiUrl: env.TURBO_API_URL,
-      },
-      null,
-      2,
-    ),
-    'utf-8',
-  );
-
-  console.log('[turborepo-setup] Config created.');
-}
-
-main();

turbo.json

@@ -28,7 +28,12 @@
     },
     "@hive/app#build": {
       "dependsOn": ["^build"],
-      "outputs": ["dist/**"]
+      "outputs": ["dist/**"],
+      "cache": false
+    },
+    "@hive/docs#build": {
+      "dependsOn": ["^build"],
+      "outputs": [".next/**", "!.next/cache/**"]
     },
     "@hive/migrations#build": {
       "dependsOn": ["^build", "@graphql-hive/client#build"],