feat(ci): add changeset package validation (#7347)

.github/workflows/changeset-validation.yaml

@@ -0,0 +1,18 @@
+on:
+  workflow_call:
+
+jobs:
+  validate-changesets:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+      - name: setup environment
+        uses: ./.github/actions/setup
+        with:
+          actor: changeset-validation
+
+      - name: validate changeset packages
+        run: pnpm tsx scripts/validate-changesets.ts

.github/workflows/pr.yaml

@@ -61,6 +61,11 @@ jobs:
     with:
       imageTag: ${{ github.event.pull_request.head.sha }}
 
+  # Changeset Validation
+  # Validates that changesets reference valid packages that exist in the monorepo
+  changeset-validation:
+    uses: ./.github/workflows/changeset-validation.yaml
+
   # ESLint and Prettier
   code-style:
     uses: ./.github/workflows/lint.yaml

scripts/validate-changesets.spec.ts

@@ -0,0 +1,222 @@
+import { describe, expect, it } from 'vitest';
+import {
+  isPackageIgnored,
+  parseChangesetFrontmatter,
+  parsePackageEntries,
+  validateChangeset,
+} from './validate-changesets';
+
+describe('validate-changesets', () => {
+  describe('parseChangesetFrontmatter', () => {
+    it('parses valid frontmatter', () => {
+      const content = `---
+'@graphql-hive/cli': patch
+---
+
+Some description here.`;
+      expect(parseChangesetFrontmatter(content)).toBe("'@graphql-hive/cli': patch");
+    });
+
+    it('parses multi-package frontmatter', () => {
+      const content = `---
+'@graphql-hive/cli': patch
+'@graphql-hive/core': minor
+---
+
+Some description here.`;
+      expect(parseChangesetFrontmatter(content)).toBe(
+        "'@graphql-hive/cli': patch\n'@graphql-hive/core': minor",
+      );
+    });
+
+    it('returns null for invalid frontmatter', () => {
+      const content = `No frontmatter here`;
+      expect(parseChangesetFrontmatter(content)).toBeNull();
+    });
+
+    it('returns null for unclosed frontmatter', () => {
+      const content = `---
+'@graphql-hive/cli': patch
+Some description here.`;
+      expect(parseChangesetFrontmatter(content)).toBeNull();
+    });
+
+    it('returns empty string for whitespace-only frontmatter', () => {
+      const content = `---
+
+---
+
+Some description here.`;
+      expect(parseChangesetFrontmatter(content)).toBe('');
+    });
+
+    it('returns null for empty frontmatter (no newline between markers)', () => {
+      const content = `---
+---
+
+Some description here.`;
+      expect(parseChangesetFrontmatter(content)).toBeNull();
+    });
+  });
+
+  describe('parsePackageEntries', () => {
+    it('parses single-quoted package names', () => {
+      const frontmatter = "'@graphql-hive/cli': patch";
+      expect(parsePackageEntries(frontmatter)).toEqual([
+        { packageName: '@graphql-hive/cli', bumpType: 'patch' },
+      ]);
+    });
+
+    it('parses double-quoted package names', () => {
+      const frontmatter = '"@graphql-hive/cli": minor';
+      expect(parsePackageEntries(frontmatter)).toEqual([
+        { packageName: '@graphql-hive/cli', bumpType: 'minor' },
+      ]);
+    });
+
+    it('parses unquoted package names', () => {
+      const frontmatter = 'hive: major';
+      expect(parsePackageEntries(frontmatter)).toEqual([
+        { packageName: 'hive', bumpType: 'major' },
+      ]);
+    });
+
+    it('parses multiple packages', () => {
+      const frontmatter = `'@graphql-hive/cli': patch
+'@graphql-hive/core': minor
+hive: major`;
+      expect(parsePackageEntries(frontmatter)).toEqual([
+        { packageName: '@graphql-hive/cli', bumpType: 'patch' },
+        { packageName: '@graphql-hive/core', bumpType: 'minor' },
+        { packageName: 'hive', bumpType: 'major' },
+      ]);
+    });
+
+    it('returns error for invalid lines', () => {
+      const frontmatter = 'invalid line without colon';
+      expect(parsePackageEntries(frontmatter)).toEqual([
+        { error: 'parse_error', line: 'invalid line without colon' },
+      ]);
+    });
+
+    it('returns error for invalid bump type', () => {
+      const frontmatter = "'@graphql-hive/cli': invalid";
+      expect(parsePackageEntries(frontmatter)).toEqual([
+        { error: 'parse_error', line: "'@graphql-hive/cli': invalid" },
+      ]);
+    });
+  });
+
+  describe('isPackageIgnored', () => {
+    const ignorePatterns = ['@hive/*', 'integration-tests', 'eslint-plugin-hive'];
+
+    it('matches exact package names', () => {
+      expect(isPackageIgnored('integration-tests', ignorePatterns)).toBe(true);
+      expect(isPackageIgnored('eslint-plugin-hive', ignorePatterns)).toBe(true);
+    });
+
+    it('matches glob patterns', () => {
+      expect(isPackageIgnored('@hive/api', ignorePatterns)).toBe(true);
+      expect(isPackageIgnored('@hive/storage', ignorePatterns)).toBe(true);
+      expect(isPackageIgnored('@hive/anything', ignorePatterns)).toBe(true);
+    });
+
+    it('does not match non-ignored packages', () => {
+      expect(isPackageIgnored('@graphql-hive/cli', ignorePatterns)).toBe(false);
+      expect(isPackageIgnored('hive', ignorePatterns)).toBe(false);
+      expect(isPackageIgnored('@graphql-hive/core', ignorePatterns)).toBe(false);
+    });
+  });
+
+  describe('validateChangeset', () => {
+    const validPackages = new Set(['@graphql-hive/cli', '@graphql-hive/core', 'hive', '@hive/api']);
+    const ignorePatterns = ['@hive/*', 'integration-tests'];
+
+    it('returns no errors for valid changeset', () => {
+      const content = `---
+'@graphql-hive/cli': patch
+---
+
+Fix something.`;
+      const errors = validateChangeset('test.md', content, validPackages, ignorePatterns);
+      expect(errors).toEqual([]);
+    });
+
+    it('returns no errors for multiple valid packages', () => {
+      const content = `---
+'@graphql-hive/cli': patch
+'@graphql-hive/core': minor
+---
+
+Fix something.`;
+      const errors = validateChangeset('test.md', content, validPackages, ignorePatterns);
+      expect(errors).toEqual([]);
+    });
+
+    it('returns error for non-existent package', () => {
+      const content = `---
+'non-existent-package': patch
+---
+
+Fix something.`;
+      const errors = validateChangeset('test.md', content, validPackages, ignorePatterns);
+      expect(errors).toHaveLength(1);
+      expect(errors[0].file).toBe('test.md');
+      expect(errors[0].message).toContain('does not exist in the monorepo');
+    });
+
+    it('returns error for ignored package', () => {
+      const content = `---
+'@hive/api': patch
+---
+
+Fix something.`;
+      const errors = validateChangeset('test.md', content, validPackages, ignorePatterns);
+      expect(errors).toHaveLength(1);
+      expect(errors[0].file).toBe('test.md');
+      expect(errors[0].message).toContain('is in the changeset ignore list');
+    });
+
+    it('returns error for invalid frontmatter', () => {
+      const content = `No frontmatter here`;
+      const errors = validateChangeset('test.md', content, validPackages, ignorePatterns);
+      expect(errors).toHaveLength(1);
+      expect(errors[0].message).toBe('Could not parse frontmatter');
+    });
+
+    it('returns error for whitespace-only frontmatter', () => {
+      const content = `---
+
+---
+
+Fix something.`;
+      const errors = validateChangeset('test.md', content, validPackages, ignorePatterns);
+      expect(errors).toHaveLength(1);
+      expect(errors[0].message).toBe('Changeset has no packages listed');
+    });
+
+    it('returns error for unparseable line', () => {
+      const content = `---
+invalid line
+---
+
+Fix something.`;
+      const errors = validateChangeset('test.md', content, validPackages, ignorePatterns);
+      expect(errors).toHaveLength(1);
+      expect(errors[0].message).toContain('Could not parse line');
+    });
+
+    it('returns multiple errors when applicable', () => {
+      const content = `---
+'non-existent': patch
+'@hive/api': minor
+---
+
+Fix something.`;
+      const errors = validateChangeset('test.md', content, validPackages, ignorePatterns);
+      expect(errors).toHaveLength(2);
+      expect(errors[0].message).toContain('does not exist');
+      expect(errors[1].message).toContain('ignore list');
+    });
+  });
+});

scripts/validate-changesets.ts

@@ -0,0 +1,235 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { getPackages } from '@manypkg/get-packages';
+
+const ROOT_DIR = path.resolve(import.meta.dirname, '..');
+const CHANGESET_DIR = path.join(ROOT_DIR, '.changeset');
+const CHANGESET_CONFIG = path.join(CHANGESET_DIR, 'config.json');
+
+export interface ValidationError {
+  file: string;
+  message: string;
+}
+
+export function parseChangesetFrontmatter(content: string): string | null {
+  const match = content.match(/^---\n([\s\S]*?)\n---/);
+  return match ? match[1] : null;
+}
+
+export function parsePackageEntries(
+  frontmatter: string,
+): Array<{ packageName: string; bumpType: string } | { error: string; line: string }> {
+  const packageLines = frontmatter.split('\n').filter(line => line.trim());
+  const results: Array<
+    { packageName: string; bumpType: string } | { error: string; line: string }
+  > = [];
+
+  for (const line of packageLines) {
+    const packageMatch = line.match(/^['"]?([^'":\s]+)['"]?\s*:\s*(patch|minor|major)$/);
+    if (!packageMatch) {
+      results.push({ error: 'parse_error', line });
+    } else {
+      results.push({ packageName: packageMatch[1], bumpType: packageMatch[2] });
+    }
+  }
+
+  return results;
+}
+
+export function isPackageIgnored(packageName: string, ignorePatterns: string[]): boolean {
+  return ignorePatterns.some(pattern => {
+    if (pattern.endsWith('*')) {
+      const prefix = pattern.slice(0, -1);
+      return packageName.startsWith(prefix);
+    }
+    return packageName === pattern;
+  });
+}
+
+export function validateChangeset(
+  fileName: string,
+  content: string,
+  validPackageNames: Set<string>,
+  ignorePatterns: string[],
+): ValidationError[] {
+  const errors: ValidationError[] = [];
+
+  const frontmatter = parseChangesetFrontmatter(content);
+  if (frontmatter === null) {
+    errors.push({ file: fileName, message: 'Could not parse frontmatter' });
+    return errors;
+  }
+
+  if (!frontmatter.trim()) {
+    errors.push({ file: fileName, message: 'Changeset has no packages listed' });
+    return errors;
+  }
+
+  const entries = parsePackageEntries(frontmatter);
+
+  if (entries.length === 0) {
+    errors.push({ file: fileName, message: 'Changeset has no packages listed' });
+    return errors;
+  }
+
+  for (const entry of entries) {
+    if ('error' in entry) {
+      errors.push({ file: fileName, message: `Could not parse line: ${entry.line}` });
+      continue;
+    }
+
+    const { packageName } = entry;
+
+    if (!validPackageNames.has(packageName)) {
+      errors.push({
+        file: fileName,
+        message:
+          `Package "${packageName}" does not exist in the monorepo.\n` +
+          `  Valid packages are:\n${Array.from(validPackageNames)
+            .sort()
+            .map(p => `    - ${p}`)
+            .join('\n')}`,
+      });
+      continue;
+    }
+
+    if (isPackageIgnored(packageName, ignorePatterns)) {
+      errors.push({
+        file: fileName,
+        message:
+          `Package "${packageName}" is in the changeset ignore list.\n` +
+          `  Ignored patterns: ${ignorePatterns.join(', ')}\n` +
+          `  This package doesn't need a changeset entry.`,
+      });
+    }
+  }
+
+  return errors;
+}
+
+function readConfig(): { ignore: string[] } {
+  let configContent: string;
+  try {
+    configContent = fs.readFileSync(CHANGESET_CONFIG, 'utf-8');
+  } catch (err) {
+    const nodeErr = err as NodeJS.ErrnoException;
+    if (nodeErr.code === 'ENOENT') {
+      console.error(`Changeset config not found at ${CHANGESET_CONFIG}.`);
+    } else {
+      console.error(`Failed to read changeset config: ${nodeErr.message}`);
+    }
+    process.exit(1);
+  }
+
+  let config: unknown;
+  try {
+    config = JSON.parse(configContent);
+  } catch {
+    console.error(
+      `Failed to parse changeset config at ${CHANGESET_CONFIG}.\n` +
+        `The JSON appears to be malformed.`,
+    );
+    process.exit(1);
+  }
+
+  if (typeof config !== 'object' || config === null) {
+    console.error(`Invalid changeset config: expected an object.`);
+    process.exit(1);
+  }
+
+  const configObj = config as Record<string, unknown>;
+  const ignore = configObj.ignore;
+
+  if (ignore !== undefined) {
+    if (!Array.isArray(ignore) || !ignore.every(item => typeof item === 'string')) {
+      console.error(
+        `Invalid changeset config: "ignore" must be an array of strings.\n` +
+          `Check ${CHANGESET_CONFIG}.`,
+      );
+      process.exit(1);
+    }
+    return { ignore };
+  }
+
+  return { ignore: [] };
+}
+
+function readChangesetFiles(): string[] {
+  let files: string[];
+  try {
+    files = fs.readdirSync(CHANGESET_DIR);
+  } catch (err) {
+    const nodeErr = err as NodeJS.ErrnoException;
+    if (nodeErr.code === 'ENOENT') {
+      console.error(`Changeset directory not found at ${CHANGESET_DIR}.`);
+    } else {
+      console.error(`Failed to read changeset directory: ${nodeErr.message}`);
+    }
+    process.exit(1);
+  }
+
+  return files.filter(file => file.endsWith('.md') && file !== 'README.md');
+}
+
+async function main() {
+  let packages: Awaited<ReturnType<typeof getPackages>>['packages'];
+  try {
+    ({ packages } = await getPackages(ROOT_DIR));
+  } catch (err) {
+    console.error(
+      `Failed to discover monorepo packages from ${ROOT_DIR}.\n` +
+        `Ensure package.json exists and workspace configuration is valid.\n` +
+        `Error: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    process.exit(1);
+  }
+
+  const validPackageNames = new Set(packages.map(pkg => pkg.packageJson.name));
+  const config = readConfig();
+  const changesetFiles = readChangesetFiles();
+
+  if (changesetFiles.length === 0) {
+    console.log('No changesets found.');
+    process.exit(0);
+  }
+
+  const allErrors: ValidationError[] = [];
+
+  for (const file of changesetFiles) {
+    const filePath = path.join(CHANGESET_DIR, file);
+    let content: string;
+    try {
+      content = fs.readFileSync(filePath, 'utf-8');
+    } catch (err) {
+      allErrors.push({
+        file,
+        message: `Could not read file: ${err instanceof Error ? err.message : String(err)}`,
+      });
+      continue;
+    }
+    const errors = validateChangeset(file, content, validPackageNames, config.ignore);
+    allErrors.push(...errors);
+  }
+
+  if (allErrors.length > 0) {
+    console.error('Changeset validation failed:\n');
+    for (const error of allErrors) {
+      console.error(`${error.file}: ${error.message}\n`);
+    }
+    process.exit(1);
+  }
+
+  console.log(`All ${changesetFiles.length} changesets validated successfully.`);
+}
+
+// Only run main when executed directly, not when imported for testing
+const isMain = process.argv[1] === import.meta.filename;
+if (isMain) {
+  main().catch(err => {
+    console.error(
+      `Unexpected error during changeset validation:\n` +
+        `${err instanceof Error ? err.message : String(err)}`,
+    );
+    process.exit(1);
+  });
+}