diff --git a/.github/workflows/db-types-diff.yaml b/.github/workflows/db-types-diff.yaml index e5f38d641..065c54d17 100644 --- a/.github/workflows/db-types-diff.yaml +++ b/.github/workflows/db-types-diff.yaml @@ -34,11 +34,11 @@ jobs: codegen: false # no need to run codegen in this case, we can skip - name: create database - working-directory: packages/services/storage + working-directory: packages/migrations run: pnpm db:create - name: migrate database - working-directory: packages/services/storage + working-directory: packages/migrations run: pnpm db:migrator up - name: generate database types diff --git a/.gitignore b/.gitignore index 4a3073773..16d128758 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,9 @@ npm-shrinkwrap.json # bob .bob/ +# Docker temp volumes .hive +.hive-dev +.volumes cypress diff --git a/packages/services/storage/configs/clickhouse/config.xml b/configs/clickhouse/config.xml similarity index 100% rename from packages/services/storage/configs/clickhouse/config.xml rename to configs/clickhouse/config.xml diff --git a/docker-compose.community.yml b/docker-compose.community.yml index afffff79d..0be11dfc7 100644 --- a/docker-compose.community.yml +++ b/docker-compose.community.yml @@ -170,7 +170,7 @@ services: - '8083:8083' command: caddy reverse-proxy --from :8083 --to s3:9000 --change-host-header - storage: + migrations: image: '${DOCKER_REGISTRY}storage${DOCKER_TAG}' networks: - 'stack' @@ -202,7 +202,7 @@ services: condition: service_healthy clickhouse: condition: service_healthy - storage: + migrations: condition: service_completed_successfully s3_provision_buckets: condition: service_completed_successfully @@ -270,7 +270,7 @@ services: networks: - 'stack' depends_on: - storage: + migrations: condition: service_completed_successfully environment: NODE_ENV: production diff --git a/packages/services/storage/docker-compose.yml b/docker-compose.dev.yml similarity index 92% rename from packages/services/storage/docker-compose.yml rename to docker-compose.dev.yml index e2e943929..e2b396423 100644 --- a/packages/services/storage/docker-compose.yml +++ b/docker-compose.dev.yml @@ -15,7 +15,7 @@ services: POSTGRES_DB: registry PGDATA: /var/lib/postgresql/data volumes: - - ./volumes/postgresql/db:/var/lib/postgresql/data + - ./.hive-dev/postgresql/db:/var/lib/postgresql/data ports: - '5432:5432' @@ -35,7 +35,7 @@ services: ports: - '6379:6379' volumes: - - './volumes/redis/db:/bitnami/redis/data' + - './.hive-dev/redis/db:/bitnami/redis/data' s3: image: quay.io/minio/minio:RELEASE.2022-11-29T23-40-49Z @@ -77,8 +77,8 @@ services: timeout: 5s retries: 3 volumes: - - ./volumes/clickhouse/logs:/var/log/clickhouse-server - - ./volumes/clickhouse/db:/var/lib/clickhouse + - ./.hive-dev/clickhouse/logs:/var/log/clickhouse-server + - ./.hive-dev/clickhouse/db:/var/lib/clickhouse - ./configs/clickhouse:/etc/clickhouse-server/conf.d ports: - '8123:8123' @@ -110,7 +110,7 @@ services: mem_limit: 300m mem_reservation: 100m volumes: - - ./volumes/broker/db:/var/lib/redpanda/data + - ./.hive-dev/broker/db:/var/lib/kafka/data supertokens: image: registry.supertokens.io/supertokens/supertokens-postgresql:4.3 diff --git a/docker.hcl b/docker.hcl index a81d9c8fd..473c01bec 100644 --- a/docker.hcl +++ b/docker.hcl @@ -48,6 +48,13 @@ function "image_tag" { result = notequal("", tag) ? "${DOCKER_REGISTRY}${name}:${tag}" : "" } +target "migrations-base" { + dockerfile = "${PWD}/migrations.dockerfile" + args = { + RELEASE = "${RELEASE}" + } +} + target "service-base" { dockerfile = "${PWD}/services.dockerfile" args = { @@ -144,11 +151,11 @@ target "server" { } target "storage" { - inherits = ["service-base", get_target()] - context = "${PWD}/packages/services/storage/dist" + inherits = ["migrations-base", get_target()] + context = "${PWD}/packages/migrations/migrations-artifact" args = { IMAGE_TITLE = "graphql-hive/storage" - IMAGE_DESCRIPTION = "The storage service of the GraphQL Hive project." + IMAGE_DESCRIPTION = "The migrations service of the GraphQL Hive project." } tags = [ local_image_tag("storage"), diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index b890da8b4..86c4c4824 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -24,7 +24,7 @@ CDN_AUTH_PRIVATE_KEY=$(openssl rand -hex 16) ``` - Run `pnpm i` at the root to install all the dependencies and run the hooks -- Run `pnpm run --filter @hive/storage setup` to create local databases +- Run `pnpm local:setup` to run Docker compose dependencies, create databases and migrate database - Run `pnpm generate` to generate the typings from the graphql files (use `pnpm graphql:generate` if you only need to run GraphQL Codegen) - Run `pnpm build` to build all services diff --git a/integration-tests/docker-compose.integration.yaml b/integration-tests/docker-compose.integration.yaml index 754e514a0..70b57d9ca 100644 --- a/integration-tests/docker-compose.integration.yaml +++ b/integration-tests/docker-compose.integration.yaml @@ -108,7 +108,7 @@ services: depends_on: clickhouse: condition: service_healthy - storage: + migrations: condition: service_completed_successfully environment: NODE_ENV: production @@ -130,7 +130,7 @@ services: depends_on: clickhouse: condition: service_healthy - storage: + migrations: condition: service_completed_successfully usage-estimator: condition: service_healthy diff --git a/migrations.dockerfile b/migrations.dockerfile new file mode 100644 index 000000000..b96ad50b2 --- /dev/null +++ b/migrations.dockerfile @@ -0,0 +1,9 @@ +FROM node:18.13.0-slim + +WORKDIR /usr/src/app +COPY . /usr/src/app/ + +ENV ENVIRONMENT production +ENV NODE_ENV production + +CMD ["npm", "run", "db:migrator", "up"] diff --git a/package.json b/package.json index 5636a70ea..d778e7e60 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "lint": "eslint --cache --ignore-path .gitignore \"{packages,cypress}/**/*.{ts,tsx}\"", "lint:fix": "pnpm lint --fix", "lint:prettier": "prettier --cache --check .", + "local:setup": "docker-compose -f docker-compose.dev.yml up -d --remove-orphans && pnpm --filter @hive/migrations db:init", "postinstall": "node ./scripts/patch-manifests.js && pnpm env:sync && node ./scripts/turborepo-cleanup.js && node ./scripts/turborepo-setup.js", "pre-commit": "exit 0 && lint-staged", "prepare": "husky install", @@ -39,7 +40,6 @@ "prettier": "prettier --cache --write --list-different .", "release": "pnpm build:libraries && changeset publish", "seed": "node scripts/seed-local-env.js", - "setup": "pnpm --filter @hive/storage setup", "test": "jest", "test:e2e": "CYPRESS_BASE_URL=$HIVE_APP_BASE_URL cypress run", "turbo": "env-cmd --silent turbo run", @@ -119,7 +119,8 @@ "@graphql-inspector/core@3.4.0": "patches/@graphql-inspector__core@3.4.0.patch", "@oclif/core@1.23.0": "patches/@oclif__core@1.23.0.patch", "bullmq@3.5.6": "patches/bullmq@3.5.6.patch", - "oclif@3.4.3": "patches/oclif@3.4.3.patch" + "oclif@3.4.3": "patches/oclif@3.4.3.patch", + "@slonik/migrator@0.8.5": "patches/@slonik__migrator@0.8.5.patch" } } } diff --git a/packages/migrations/.env.template b/packages/migrations/.env.template new file mode 100644 index 000000000..3d9bc4565 --- /dev/null +++ b/packages/migrations/.env.template @@ -0,0 +1,13 @@ +MIGRATOR="up" +CLICKHOUSE_MIGRATOR="up" +CLICKHOUSE_PROTOCOL="http" +CLICKHOUSE_HOST="localhost" +CLICKHOUSE_PORT="8123" +CLICKHOUSE_USERNAME="test" +CLICKHOUSE_PASSWORD="test" + +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_DB=registry \ No newline at end of file diff --git a/packages/migrations/.gitignore b/packages/migrations/.gitignore new file mode 100644 index 000000000..86431e0e8 --- /dev/null +++ b/packages/migrations/.gitignore @@ -0,0 +1 @@ +migrations-artifact \ No newline at end of file diff --git a/packages/migrations/README.md b/packages/migrations/README.md new file mode 100644 index 000000000..4859d0cc6 --- /dev/null +++ b/packages/migrations/README.md @@ -0,0 +1,57 @@ +# `@hive/migrations` + +Service for creating, managing and running database migrations. + +We are using [Slonik Migrator](https://github.com/mmkal/slonik-tools/tree/main/packages/migrator) +under the hood, to generate, manage, track and run migrations. + +## Development Workflow + +To add new migrations, please follow the instructions below, based on the type of the migration you +need to implement. + +Please follow these guidelines: + +1. Do not delete previous migration files! +2. Do not change any existing migration code or file name, once it's merged to `main` branch! +3. If you need a change in database or structure, make sure that the `down` migration does the exact + opposite (usually, in the reversed order). +4. Migrations should not take too long to run - if your migration is doing too many things, please + consider to break it to smaller pieces, or rethink your implemenation. +5. **Remember**: the database in production always contains more data: a tiny migration on + development might take a long time to complete on production environment! + +> Please refer to the `package.json` of this package for useful scripts that might make it simpler +> for you to write migrations. + +## Adding new Postgres migrations + +For postgres migrations, just run the following command with your configuration: + +``` +pnpm migration:create --name "do_something.sql" +``` + +Then, make sure to write both `up` and `down` migrations, based on your need. + +## ClickHouse Migrations + +We aim to avoid ClickHouse data migrations, as they are heavy and complicated to apply. + +If you need to apply ClickHouse database structure changes, please try to add and avoid changes to +existing objects. + +Follow `src/clickhouse.ts` for more information and the detailed database structure. + +## Custom Migrations + +You are also able to use TypeScript to write and apply migrations. + +To create a custom migration, use the following command: + +``` +pnpm migration:create --name "do_something.mts" --allow-extension ".mts" +``` + +> If you need external dependencies at runtime, please make sure to add your dependencies under +> `dependencies` in `package.json`. diff --git a/packages/migrations/package.json b/packages/migrations/package.json new file mode 100644 index 000000000..6060c860a --- /dev/null +++ b/packages/migrations/package.json @@ -0,0 +1,32 @@ +{ + "name": "@hive/migrations", + "version": "0.0.0", + "type": "module", + "license": "MIT", + "private": true, + "scripts": { + "build": "pnpm --filter=@hive/migrations deploy ./migrations-artifact/", + "db:create": "node tools/create-db.mjs", + "db:init": "pnpm db:create && pnpm migration:run", + "db:migrator": "node --experimental-specifier-resolution=node --loader ts-node/esm src/index.ts", + "migration:create": "pnpm db:migrator create", + "migration:rollback": "pnpm db:migrator down", + "migration:run": "pnpm db:migrator up" + }, + "dependencies": { + "@slonik/migrator": "0.8.5", + "dotenv": "16.0.3", + "got": "12.5.3", + "slonik": "30.1.2", + "ts-node": "10.9.1", + "zod": "3.20.2" + }, + "devDependencies": { + "@types/node": "18.11.18", + "@types/pg": "8.6.6", + "copyfiles": "2.4.1", + "pg-promise": "10.15.4", + "tslib": "2.4.1", + "typescript": "4.9.4" + } +} diff --git a/packages/services/storage/migrations/actions/2021-03-05T19-06-23.initial.sql b/packages/migrations/src/actions/2021-03-05T19-06-23.initial.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-03-05T19-06-23.initial.sql rename to packages/migrations/src/actions/2021-03-05T19-06-23.initial.sql diff --git a/packages/services/storage/migrations/actions/2021-03-08T11-02-26.urls.sql b/packages/migrations/src/actions/2021-03-08T11-02-26.urls.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-03-08T11-02-26.urls.sql rename to packages/migrations/src/actions/2021-03-08T11-02-26.urls.sql diff --git a/packages/services/storage/migrations/actions/2021-03-09T10-30-35.roles.sql b/packages/migrations/src/actions/2021-03-09T10-30-35.roles.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-03-09T10-30-35.roles.sql rename to packages/migrations/src/actions/2021-03-09T10-30-35.roles.sql diff --git a/packages/services/storage/migrations/actions/2021-03-09T14-02-34.activities.sql b/packages/migrations/src/actions/2021-03-09T14-02-34.activities.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-03-09T14-02-34.activities.sql rename to packages/migrations/src/actions/2021-03-09T14-02-34.activities.sql diff --git a/packages/services/storage/migrations/actions/2021-03-15T19-32-01.commit-project-id.sql b/packages/migrations/src/actions/2021-03-15T19-32-01.commit-project-id.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-03-15T19-32-01.commit-project-id.sql rename to packages/migrations/src/actions/2021-03-15T19-32-01.commit-project-id.sql diff --git a/packages/services/storage/migrations/actions/2021-04-20T11-30-30.tokens.sql b/packages/migrations/src/actions/2021-04-20T11-30-30.tokens.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-04-20T11-30-30.tokens.sql rename to packages/migrations/src/actions/2021-04-20T11-30-30.tokens.sql diff --git a/packages/services/storage/migrations/actions/2021-04-30T07-01-57.token-per-target.sql b/packages/migrations/src/actions/2021-04-30T07-01-57.token-per-target.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-04-30T07-01-57.token-per-target.sql rename to packages/migrations/src/actions/2021-04-30T07-01-57.token-per-target.sql diff --git a/packages/services/storage/migrations/actions/2021-04-30T11-47-26.validation.sql b/packages/migrations/src/actions/2021-04-30T11-47-26.validation.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-04-30T11-47-26.validation.sql rename to packages/migrations/src/actions/2021-04-30T11-47-26.validation.sql diff --git a/packages/services/storage/migrations/actions/2021-04-30T18-30-00.persisted-operations.sql b/packages/migrations/src/actions/2021-04-30T18-30-00.persisted-operations.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-04-30T18-30-00.persisted-operations.sql rename to packages/migrations/src/actions/2021-04-30T18-30-00.persisted-operations.sql diff --git a/packages/services/storage/migrations/actions/2021-05-07T07-28-07.token-last-used-at.sql b/packages/migrations/src/actions/2021-05-07T07-28-07.token-last-used-at.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-05-07T07-28-07.token-last-used-at.sql rename to packages/migrations/src/actions/2021-05-07T07-28-07.token-last-used-at.sql diff --git a/packages/services/storage/migrations/actions/2021-06-11T10-46-24.slack-integration.sql b/packages/migrations/src/actions/2021-06-11T10-46-24.slack-integration.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-06-11T10-46-24.slack-integration.sql rename to packages/migrations/src/actions/2021-06-11T10-46-24.slack-integration.sql diff --git a/packages/services/storage/migrations/actions/2021-06-11T15-38-28.alerts.sql b/packages/migrations/src/actions/2021-06-11T15-38-28.alerts.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-06-11T15-38-28.alerts.sql rename to packages/migrations/src/actions/2021-06-11T15-38-28.alerts.sql diff --git a/packages/services/storage/migrations/actions/2021-08-18T13-20-45.urls.sql b/packages/migrations/src/actions/2021-08-18T13-20-45.urls.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-08-18T13-20-45.urls.sql rename to packages/migrations/src/actions/2021-08-18T13-20-45.urls.sql diff --git a/packages/services/storage/migrations/actions/2021-08-27T14-19-48.non-unique-emails.sql b/packages/migrations/src/actions/2021-08-27T14-19-48.non-unique-emails.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021-08-27T14-19-48.non-unique-emails.sql rename to packages/migrations/src/actions/2021-08-27T14-19-48.non-unique-emails.sql diff --git a/packages/services/storage/migrations/actions/2021.09.17T14.45.36.token-deleted.sql b/packages/migrations/src/actions/2021.09.17T14.45.36.token-deleted.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021.09.17T14.45.36.token-deleted.sql rename to packages/migrations/src/actions/2021.09.17T14.45.36.token-deleted.sql diff --git a/packages/services/storage/migrations/actions/2021.10.07T12.11.13.access-scopes.sql b/packages/migrations/src/actions/2021.10.07T12.11.13.access-scopes.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021.10.07T12.11.13.access-scopes.sql rename to packages/migrations/src/actions/2021.10.07T12.11.13.access-scopes.sql diff --git a/packages/services/storage/migrations/actions/2021.11.22T11.23.44.base-schema.sql b/packages/migrations/src/actions/2021.11.22T11.23.44.base-schema.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021.11.22T11.23.44.base-schema.sql rename to packages/migrations/src/actions/2021.11.22T11.23.44.base-schema.sql diff --git a/packages/services/storage/migrations/actions/2021.12.20T14.05.30.commits-with-targets.sql b/packages/migrations/src/actions/2021.12.20T14.05.30.commits-with-targets.sql similarity index 100% rename from packages/services/storage/migrations/actions/2021.12.20T14.05.30.commits-with-targets.sql rename to packages/migrations/src/actions/2021.12.20T14.05.30.commits-with-targets.sql diff --git a/packages/services/storage/migrations/actions/2022.01.21T12.34.46.validation-targets.sql b/packages/migrations/src/actions/2022.01.21T12.34.46.validation-targets.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.01.21T12.34.46.validation-targets.sql rename to packages/migrations/src/actions/2022.01.21T12.34.46.validation-targets.sql diff --git a/packages/services/storage/migrations/actions/2022.03.28T10.31.26.github-integration.sql b/packages/migrations/src/actions/2022.03.28T10.31.26.github-integration.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.03.28T10.31.26.github-integration.sql rename to packages/migrations/src/actions/2022.03.28T10.31.26.github-integration.sql diff --git a/packages/services/storage/migrations/actions/2022.04.15T14.24.17.hash-tokens.sql b/packages/migrations/src/actions/2022.04.15T14.24.17.hash-tokens.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.04.15T14.24.17.hash-tokens.sql rename to packages/migrations/src/actions/2022.04.15T14.24.17.hash-tokens.sql diff --git a/packages/services/storage/migrations/actions/2022.05.03T15.58.13.org_rate_limits.sql b/packages/migrations/src/actions/2022.05.03T15.58.13.org_rate_limits.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.05.03T15.58.13.org_rate_limits.sql rename to packages/migrations/src/actions/2022.05.03T15.58.13.org_rate_limits.sql diff --git a/packages/services/storage/migrations/actions/2022.05.04T11.01.22.billing_plans.sql b/packages/migrations/src/actions/2022.05.04T11.01.22.billing_plans.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.05.04T11.01.22.billing_plans.sql rename to packages/migrations/src/actions/2022.05.04T11.01.22.billing_plans.sql diff --git a/packages/services/storage/migrations/actions/2022.05.05T08.05.35.commits-metadata.sql b/packages/migrations/src/actions/2022.05.05T08.05.35.commits-metadata.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.05.05T08.05.35.commits-metadata.sql rename to packages/migrations/src/actions/2022.05.05T08.05.35.commits-metadata.sql diff --git a/packages/services/storage/migrations/actions/2022.07.07T12.15.10.no-schema-pushes-limit.sql b/packages/migrations/src/actions/2022.07.07T12.15.10.no-schema-pushes-limit.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.07.07T12.15.10.no-schema-pushes-limit.sql rename to packages/migrations/src/actions/2022.07.07T12.15.10.no-schema-pushes-limit.sql diff --git a/packages/services/storage/migrations/actions/2022.07.11T10.09.41.get-started-wizard.sql b/packages/migrations/src/actions/2022.07.11T10.09.41.get-started-wizard.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.07.11T10.09.41.get-started-wizard.sql rename to packages/migrations/src/actions/2022.07.11T10.09.41.get-started-wizard.sql diff --git a/packages/services/storage/migrations/actions/2022.07.11T20.09.37.migrate-pro-hobby-retention.sql b/packages/migrations/src/actions/2022.07.11T20.09.37.migrate-pro-hobby-retention.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.07.11T20.09.37.migrate-pro-hobby-retention.sql rename to packages/migrations/src/actions/2022.07.11T20.09.37.migrate-pro-hobby-retention.sql diff --git a/packages/services/storage/migrations/actions/2022.07.18T10.10.44.target-validation-client-exclusion.sql b/packages/migrations/src/actions/2022.07.18T10.10.44.target-validation-client-exclusion.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.07.18T10.10.44.target-validation-client-exclusion.sql rename to packages/migrations/src/actions/2022.07.18T10.10.44.target-validation-client-exclusion.sql diff --git a/packages/services/storage/migrations/actions/2022.08.25T09.59.16.multiple-invitation-codes.sql b/packages/migrations/src/actions/2022.08.25T09.59.16.multiple-invitation-codes.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.08.25T09.59.16.multiple-invitation-codes.sql rename to packages/migrations/src/actions/2022.08.25T09.59.16.multiple-invitation-codes.sql diff --git a/packages/services/storage/migrations/actions/2022.08.26T06.23.24.add-supertokens-id.sql b/packages/migrations/src/actions/2022.08.26T06.23.24.add-supertokens-id.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.08.26T06.23.24.add-supertokens-id.sql rename to packages/migrations/src/actions/2022.08.26T06.23.24.add-supertokens-id.sql diff --git a/packages/services/storage/migrations/actions/2022.09.14T16.09.43.external-projects.sql b/packages/migrations/src/actions/2022.09.14T16.09.43.external-projects.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.09.14T16.09.43.external-projects.sql rename to packages/migrations/src/actions/2022.09.14T16.09.43.external-projects.sql diff --git a/packages/services/storage/migrations/actions/2022.10.20T08.00.46.oidc-integrations.sql b/packages/migrations/src/actions/2022.10.20T08.00.46.oidc-integrations.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.10.20T08.00.46.oidc-integrations.sql rename to packages/migrations/src/actions/2022.10.20T08.00.46.oidc-integrations.sql diff --git a/packages/services/storage/migrations/actions/2022.11.07T09.30.47.user-table-varchar-to-text.sql b/packages/migrations/src/actions/2022.11.07T09.30.47.user-table-varchar-to-text.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.11.07T09.30.47.user-table-varchar-to-text.sql rename to packages/migrations/src/actions/2022.11.07T09.30.47.user-table-varchar-to-text.sql diff --git a/packages/services/storage/migrations/actions/2022.12.03T09.12.28.organization-transfer.sql b/packages/migrations/src/actions/2022.12.03T09.12.28.organization-transfer.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.12.03T09.12.28.organization-transfer.sql rename to packages/migrations/src/actions/2022.12.03T09.12.28.organization-transfer.sql diff --git a/packages/services/storage/migrations/actions/2022.12.20T09.20.36.oidc-columns.sql b/packages/migrations/src/actions/2022.12.20T09.20.36.oidc-columns.sql similarity index 100% rename from packages/services/storage/migrations/actions/2022.12.20T09.20.36.oidc-columns.sql rename to packages/migrations/src/actions/2022.12.20T09.20.36.oidc-columns.sql diff --git a/packages/services/storage/migrations/actions/2023.01.04T17.00.23.hobby-7-by-default.sql b/packages/migrations/src/actions/2023.01.04T17.00.23.hobby-7-by-default.sql similarity index 100% rename from packages/services/storage/migrations/actions/2023.01.04T17.00.23.hobby-7-by-default.sql rename to packages/migrations/src/actions/2023.01.04T17.00.23.hobby-7-by-default.sql diff --git a/packages/services/storage/migrations/actions/down/2021-03-05T19-06-23.initial.sql b/packages/migrations/src/actions/down/2021-03-05T19-06-23.initial.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-03-05T19-06-23.initial.sql rename to packages/migrations/src/actions/down/2021-03-05T19-06-23.initial.sql diff --git a/packages/services/storage/migrations/actions/down/2021-03-08T11-02-26.urls.sql b/packages/migrations/src/actions/down/2021-03-08T11-02-26.urls.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-03-08T11-02-26.urls.sql rename to packages/migrations/src/actions/down/2021-03-08T11-02-26.urls.sql diff --git a/packages/services/storage/migrations/actions/down/2021-03-09T10-30-35.roles.sql b/packages/migrations/src/actions/down/2021-03-09T10-30-35.roles.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-03-09T10-30-35.roles.sql rename to packages/migrations/src/actions/down/2021-03-09T10-30-35.roles.sql diff --git a/packages/services/storage/migrations/actions/down/2021-03-09T14-02-34.activities.sql b/packages/migrations/src/actions/down/2021-03-09T14-02-34.activities.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-03-09T14-02-34.activities.sql rename to packages/migrations/src/actions/down/2021-03-09T14-02-34.activities.sql diff --git a/packages/services/storage/migrations/actions/down/2021-03-15T19-32-01.commit-project-id.sql b/packages/migrations/src/actions/down/2021-03-15T19-32-01.commit-project-id.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-03-15T19-32-01.commit-project-id.sql rename to packages/migrations/src/actions/down/2021-03-15T19-32-01.commit-project-id.sql diff --git a/packages/services/storage/migrations/actions/down/2021-04-20T11-30-30.tokens.sql b/packages/migrations/src/actions/down/2021-04-20T11-30-30.tokens.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-04-20T11-30-30.tokens.sql rename to packages/migrations/src/actions/down/2021-04-20T11-30-30.tokens.sql diff --git a/packages/services/storage/migrations/actions/down/2021-04-30T07-01-57.token-per-target.sql b/packages/migrations/src/actions/down/2021-04-30T07-01-57.token-per-target.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-04-30T07-01-57.token-per-target.sql rename to packages/migrations/src/actions/down/2021-04-30T07-01-57.token-per-target.sql diff --git a/packages/services/storage/migrations/actions/down/2021-04-30T11-47-26.validation.sql b/packages/migrations/src/actions/down/2021-04-30T11-47-26.validation.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-04-30T11-47-26.validation.sql rename to packages/migrations/src/actions/down/2021-04-30T11-47-26.validation.sql diff --git a/packages/services/storage/migrations/actions/down/2021-04-30T18-30-00.persisted-operations.sql b/packages/migrations/src/actions/down/2021-04-30T18-30-00.persisted-operations.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-04-30T18-30-00.persisted-operations.sql rename to packages/migrations/src/actions/down/2021-04-30T18-30-00.persisted-operations.sql diff --git a/packages/services/storage/migrations/actions/down/2021-05-07T07-28-07.token-last-used-at.sql b/packages/migrations/src/actions/down/2021-05-07T07-28-07.token-last-used-at.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-05-07T07-28-07.token-last-used-at.sql rename to packages/migrations/src/actions/down/2021-05-07T07-28-07.token-last-used-at.sql diff --git a/packages/services/storage/migrations/actions/down/2021-06-11T10-46-24.slack-integration.sql b/packages/migrations/src/actions/down/2021-06-11T10-46-24.slack-integration.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-06-11T10-46-24.slack-integration.sql rename to packages/migrations/src/actions/down/2021-06-11T10-46-24.slack-integration.sql diff --git a/packages/services/storage/migrations/actions/down/2021-06-11T15-38-28.alerts.sql b/packages/migrations/src/actions/down/2021-06-11T15-38-28.alerts.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-06-11T15-38-28.alerts.sql rename to packages/migrations/src/actions/down/2021-06-11T15-38-28.alerts.sql diff --git a/packages/services/storage/migrations/actions/down/2021-08-18T13-20-45.urls.sql b/packages/migrations/src/actions/down/2021-08-18T13-20-45.urls.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-08-18T13-20-45.urls.sql rename to packages/migrations/src/actions/down/2021-08-18T13-20-45.urls.sql diff --git a/packages/services/storage/migrations/actions/down/2021-08-27T14-19-48.non-unique-emails.sql b/packages/migrations/src/actions/down/2021-08-27T14-19-48.non-unique-emails.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021-08-27T14-19-48.non-unique-emails.sql rename to packages/migrations/src/actions/down/2021-08-27T14-19-48.non-unique-emails.sql diff --git a/packages/services/storage/migrations/actions/down/2021.09.17T14.45.36.token-deleted.sql b/packages/migrations/src/actions/down/2021.09.17T14.45.36.token-deleted.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021.09.17T14.45.36.token-deleted.sql rename to packages/migrations/src/actions/down/2021.09.17T14.45.36.token-deleted.sql diff --git a/packages/services/storage/migrations/actions/down/2021.10.07T12.11.13.access-scopes.sql b/packages/migrations/src/actions/down/2021.10.07T12.11.13.access-scopes.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021.10.07T12.11.13.access-scopes.sql rename to packages/migrations/src/actions/down/2021.10.07T12.11.13.access-scopes.sql diff --git a/packages/services/storage/migrations/actions/down/2021.11.22T11.23.44.base-schema.sql b/packages/migrations/src/actions/down/2021.11.22T11.23.44.base-schema.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021.11.22T11.23.44.base-schema.sql rename to packages/migrations/src/actions/down/2021.11.22T11.23.44.base-schema.sql diff --git a/packages/services/storage/migrations/actions/down/2021.12.20T14.05.30.commits-with-targets.sql b/packages/migrations/src/actions/down/2021.12.20T14.05.30.commits-with-targets.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2021.12.20T14.05.30.commits-with-targets.sql rename to packages/migrations/src/actions/down/2021.12.20T14.05.30.commits-with-targets.sql diff --git a/packages/services/storage/migrations/actions/down/2022.01.21T12.34.46.validation-targets.sql b/packages/migrations/src/actions/down/2022.01.21T12.34.46.validation-targets.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.01.21T12.34.46.validation-targets.sql rename to packages/migrations/src/actions/down/2022.01.21T12.34.46.validation-targets.sql diff --git a/packages/services/storage/migrations/actions/down/2022.03.28T10.31.26.github-integration.sql b/packages/migrations/src/actions/down/2022.03.28T10.31.26.github-integration.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.03.28T10.31.26.github-integration.sql rename to packages/migrations/src/actions/down/2022.03.28T10.31.26.github-integration.sql diff --git a/packages/services/storage/migrations/actions/down/2022.04.15T14.24.17.hash-tokens.sql b/packages/migrations/src/actions/down/2022.04.15T14.24.17.hash-tokens.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.04.15T14.24.17.hash-tokens.sql rename to packages/migrations/src/actions/down/2022.04.15T14.24.17.hash-tokens.sql diff --git a/packages/services/storage/migrations/actions/down/2022.05.03T15.58.13.org_rate_limits.sql b/packages/migrations/src/actions/down/2022.05.03T15.58.13.org_rate_limits.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.05.03T15.58.13.org_rate_limits.sql rename to packages/migrations/src/actions/down/2022.05.03T15.58.13.org_rate_limits.sql diff --git a/packages/services/storage/migrations/actions/down/2022.05.04T11.01.22.billing_plans.sql b/packages/migrations/src/actions/down/2022.05.04T11.01.22.billing_plans.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.05.04T11.01.22.billing_plans.sql rename to packages/migrations/src/actions/down/2022.05.04T11.01.22.billing_plans.sql diff --git a/packages/services/storage/migrations/actions/down/2022.05.05T08.05.35.commits-metadata.sql b/packages/migrations/src/actions/down/2022.05.05T08.05.35.commits-metadata.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.05.05T08.05.35.commits-metadata.sql rename to packages/migrations/src/actions/down/2022.05.05T08.05.35.commits-metadata.sql diff --git a/packages/services/storage/migrations/actions/down/2022.07.07T12.15.10.no-schema-pushes-limit.sql b/packages/migrations/src/actions/down/2022.07.07T12.15.10.no-schema-pushes-limit.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.07.07T12.15.10.no-schema-pushes-limit.sql rename to packages/migrations/src/actions/down/2022.07.07T12.15.10.no-schema-pushes-limit.sql diff --git a/packages/services/storage/migrations/actions/down/2022.07.11T10.09.41.get-started-wizard.sql b/packages/migrations/src/actions/down/2022.07.11T10.09.41.get-started-wizard.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.07.11T10.09.41.get-started-wizard.sql rename to packages/migrations/src/actions/down/2022.07.11T10.09.41.get-started-wizard.sql diff --git a/packages/services/storage/migrations/actions/down/2022.07.11T20.09.37.migrate-pro-hobby-retention.sql b/packages/migrations/src/actions/down/2022.07.11T20.09.37.migrate-pro-hobby-retention.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.07.11T20.09.37.migrate-pro-hobby-retention.sql rename to packages/migrations/src/actions/down/2022.07.11T20.09.37.migrate-pro-hobby-retention.sql diff --git a/packages/services/storage/migrations/actions/down/2022.07.18T10.10.44.target-validation-client-exclusion.sql b/packages/migrations/src/actions/down/2022.07.18T10.10.44.target-validation-client-exclusion.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.07.18T10.10.44.target-validation-client-exclusion.sql rename to packages/migrations/src/actions/down/2022.07.18T10.10.44.target-validation-client-exclusion.sql diff --git a/packages/services/storage/migrations/actions/down/2022.08.25T09.59.16.multiple-invitation-codes.sql b/packages/migrations/src/actions/down/2022.08.25T09.59.16.multiple-invitation-codes.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.08.25T09.59.16.multiple-invitation-codes.sql rename to packages/migrations/src/actions/down/2022.08.25T09.59.16.multiple-invitation-codes.sql diff --git a/packages/services/storage/migrations/actions/down/2022.08.26T06.23.24.add-supertokens-id.sql b/packages/migrations/src/actions/down/2022.08.26T06.23.24.add-supertokens-id.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.08.26T06.23.24.add-supertokens-id.sql rename to packages/migrations/src/actions/down/2022.08.26T06.23.24.add-supertokens-id.sql diff --git a/packages/services/storage/migrations/actions/down/2022.09.14T16.09.43.external-projects.sql b/packages/migrations/src/actions/down/2022.09.14T16.09.43.external-projects.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.09.14T16.09.43.external-projects.sql rename to packages/migrations/src/actions/down/2022.09.14T16.09.43.external-projects.sql diff --git a/packages/services/storage/migrations/actions/down/2022.10.20T08.00.46.oidc-integrations.sql b/packages/migrations/src/actions/down/2022.10.20T08.00.46.oidc-integrations.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.10.20T08.00.46.oidc-integrations.sql rename to packages/migrations/src/actions/down/2022.10.20T08.00.46.oidc-integrations.sql diff --git a/packages/services/storage/migrations/actions/down/2022.11.07T09.30.47.user-table-varchar-to-text.sql b/packages/migrations/src/actions/down/2022.11.07T09.30.47.user-table-varchar-to-text.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.11.07T09.30.47.user-table-varchar-to-text.sql rename to packages/migrations/src/actions/down/2022.11.07T09.30.47.user-table-varchar-to-text.sql diff --git a/packages/services/storage/migrations/actions/down/2022.12.03T09.12.28.organization-transfer.sql b/packages/migrations/src/actions/down/2022.12.03T09.12.28.organization-transfer.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.12.03T09.12.28.organization-transfer.sql rename to packages/migrations/src/actions/down/2022.12.03T09.12.28.organization-transfer.sql diff --git a/packages/services/storage/migrations/actions/down/2022.12.20T09.20.36.oidc-columns.sql b/packages/migrations/src/actions/down/2022.12.20T09.20.36.oidc-columns.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2022.12.20T09.20.36.oidc-columns.sql rename to packages/migrations/src/actions/down/2022.12.20T09.20.36.oidc-columns.sql diff --git a/packages/services/storage/migrations/actions/down/2023.01.04T17.00.23.hobby-7-by-default.sql b/packages/migrations/src/actions/down/2023.01.04T17.00.23.hobby-7-by-default.sql similarity index 100% rename from packages/services/storage/migrations/actions/down/2023.01.04T17.00.23.hobby-7-by-default.sql rename to packages/migrations/src/actions/down/2023.01.04T17.00.23.hobby-7-by-default.sql diff --git a/packages/services/storage/migrations/clickhouse.ts b/packages/migrations/src/clickhouse.ts similarity index 100% rename from packages/services/storage/migrations/clickhouse.ts rename to packages/migrations/src/clickhouse.ts diff --git a/packages/migrations/src/environment.ts b/packages/migrations/src/environment.ts new file mode 100644 index 000000000..cc16bac5c --- /dev/null +++ b/packages/migrations/src/environment.ts @@ -0,0 +1,112 @@ +import { config as dotenv } from 'dotenv'; +import zod from 'zod'; + +// eslint-disable-next-line no-process-env +if (!process.env.RELEASE) { + dotenv({ + debug: true, + }); +} + +const isNumberString = (input: unknown) => zod.string().regex(/^\d+$/).safeParse(input).success; + +const numberFromNumberOrNumberString = (input: unknown): number | undefined => { + if (typeof input == 'number') return input; + if (isNumberString(input)) return Number(input); +}; + +const NumberFromString = zod.preprocess(numberFromNumberOrNumberString, zod.number().min(1)); + +// treat an empty string (`''`) as undefined +const emptyString = (input: T) => { + return zod.preprocess((value: unknown) => { + if (value === '') return undefined; + return value; + }, input); +}; + +const EnvironmentModel = zod.object({ + ENVIRONMENT: emptyString(zod.string().optional()), + RELEASE: emptyString(zod.string().optional()), + MIGRATOR: emptyString(zod.string().optional()), + CLICKHOUSE_MIGRATOR: emptyString(zod.string().optional()), +}); + +const PostgresModel = zod.object({ + POSTGRES_SSL: emptyString(zod.union([zod.literal('1'), zod.literal('0')]).optional()), + POSTGRES_HOST: zod.string(), + POSTGRES_PORT: NumberFromString, + POSTGRES_DB: zod.string(), + POSTGRES_USER: zod.string(), + POSTGRES_PASSWORD: zod.string(), +}); + +const ClickHouseModel = zod.union([ + zod.object({ + CLICKHOUSE_PROTOCOL: zod.union([zod.literal('http'), zod.literal('https')]), + CLICKHOUSE_HOST: zod.string(), + CLICKHOUSE_PORT: NumberFromString, + CLICKHOUSE_USERNAME: zod.string(), + CLICKHOUSE_PASSWORD: zod.string(), + }), + zod.object({}), +]); + +const configs = { + // eslint-disable-next-line no-process-env + base: EnvironmentModel.safeParse(process.env), + // eslint-disable-next-line no-process-env + clickhouse: ClickHouseModel.safeParse(process.env), + // eslint-disable-next-line no-process-env + postgres: PostgresModel.safeParse(process.env), +}; + +const environmentErrors: Array = []; + +for (const config of Object.values(configs)) { + if (config.success === false) { + environmentErrors.push(JSON.stringify(config.error.format(), null, 4)); + } +} + +if (environmentErrors.length) { + const fullError = environmentErrors.join(`\n`); + console.error('❌ Invalid environment variables:', fullError); + process.exit(1); +} + +function extractConfig(config: zod.SafeParseReturnType): Output { + if (!config.success) { + throw new Error('Something went wrong.'); + } + return config.data; +} + +const base = extractConfig(configs.base); +const postgres = extractConfig(configs.postgres); +const clickhouse = extractConfig(configs.clickhouse); + +export const env = { + environment: base.ENVIRONMENT, + release: base.RELEASE ?? 'local', + postgres: { + host: postgres.POSTGRES_HOST, + port: postgres.POSTGRES_PORT, + db: postgres.POSTGRES_DB, + user: postgres.POSTGRES_USER, + password: postgres.POSTGRES_PASSWORD, + ssl: postgres.POSTGRES_SSL === '1', + }, + clickhouse: + 'CLICKHOUSE_PROTOCOL' in clickhouse + ? { + protocol: clickhouse.CLICKHOUSE_PROTOCOL, + host: clickhouse.CLICKHOUSE_HOST, + port: clickhouse.CLICKHOUSE_PORT, + username: clickhouse.CLICKHOUSE_USERNAME, + password: clickhouse.CLICKHOUSE_PASSWORD, + } + : null, + isMigrator: base.MIGRATOR === 'up', + isClickHouseMigrator: base.CLICKHOUSE_MIGRATOR === 'up', +} as const; diff --git a/packages/services/storage/migrations/index.ts b/packages/migrations/src/index.ts similarity index 80% rename from packages/services/storage/migrations/index.ts rename to packages/migrations/src/index.ts index 0ef005cee..b197a241d 100644 --- a/packages/services/storage/migrations/index.ts +++ b/packages/migrations/src/index.ts @@ -3,9 +3,20 @@ import path from 'node:path'; import url from 'node:url'; import { SlonikMigrator } from '@slonik/migrator'; import { createPool } from 'slonik'; -import { createConnectionString } from '../src/db'; -import { env } from '../src/environment'; import { migrateClickHouse } from './clickhouse'; +import { env } from './environment'; + +export function createConnectionString(config: { + host: string; + port: number; + password: string; + user: string; + db: string; + ssl: boolean; +}) { + // prettier-ignore + return `postgres://${config.user}:${config.password}@${config.host}:${config.port}/${config.db}${config.ssl ? '?sslmode=require' : '?sslmode=disable'}`; +} const [, , cmd] = process.argv; const slonik = await createPool(createConnectionString(env.postgres)); diff --git a/packages/services/storage/tools/create-db.mjs b/packages/migrations/tools/create-db.mjs similarity index 100% rename from packages/services/storage/tools/create-db.mjs rename to packages/migrations/tools/create-db.mjs diff --git a/packages/services/storage/tools/db-connection-string.cjs b/packages/migrations/tools/db-connection-string.cjs similarity index 100% rename from packages/services/storage/tools/db-connection-string.cjs rename to packages/migrations/tools/db-connection-string.cjs diff --git a/packages/migrations/tsconfig.json b/packages/migrations/tsconfig.json new file mode 100644 index 000000000..fa1c551f2 --- /dev/null +++ b/packages/migrations/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "esnext", + "rootDir": "./", + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "importHelpers": true, + "allowJs": true, + "skipLibCheck": true, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "sourceMap": true, + "declaration": true, + "resolveJsonModule": true, + "moduleResolution": "node", + "strict": true + }, + "include": ["src"] +} diff --git a/packages/services/storage/.gitignore b/packages/services/storage/.gitignore index b9c89a329..a8d7c85b9 100644 --- a/packages/services/storage/.gitignore +++ b/packages/services/storage/.gitignore @@ -1,5 +1,4 @@ *.log .DS_Store node_modules -dist -volumes +dist \ No newline at end of file diff --git a/packages/services/storage/README.md b/packages/services/storage/README.md index a859bb12f..0232d9d3d 100644 --- a/packages/services/storage/README.md +++ b/packages/services/storage/README.md @@ -1,12 +1,5 @@ # `@hive/storage` -Service for running postgres and clickhouse database migrations. +Service for creating database access layer and manage data storage. -## Configuration - -| Name | Required | Description | Example Value | -| ----------------------- | -------- | ------------------------------------------------------------------------------------- | ---------------------------------------------------- | -| TODO: pg and clickhouse | **Yes** | TODO | TODO | -| `ENVIRONMENT` | No | The environment of your Hive app. (**Note:** This will be used for Sentry reporting.) | `staging` | -| `SENTRY_DSN` | No | The DSN for reporting errors to Sentry. | `https://dooobars@o557896.ingest.sentry.io/12121212` | -| `SENTRY_ENABLED` | No | Whether Sentry error reporting should be enabled. | `1` (enabled) or `0` (disabled) | +We are using Slonik as a wrapper for our Postgres instance. diff --git a/packages/services/storage/package.json b/packages/services/storage/package.json index 77d0ed578..a217b81a8 100644 --- a/packages/services/storage/package.json +++ b/packages/services/storage/package.json @@ -9,23 +9,10 @@ }, "scripts": { "build": "bob runify --single", - "db": "pnpm db:create && pnpm migration:run", - "db:create": "node tools/create-db.mjs", - "db:dev": "docker compose up --remove-orphans", - "db:generate": "schemats generate --config schemats.cjs -o src/db/types.ts", - "db:migrator": "node --experimental-specifier-resolution=node --loader ts-node/esm migrations/index.ts", - "db:start": "docker-compose up -d --remove-orphans", - "dev": "pnpm db:dev", - "migration:create": "pnpm db:migrator create", - "migration:rollback": "pnpm db:migrator down", - "migration:run": "pnpm db:migrator up", - "postbuild": "copyfiles -f \"migrations/actions/*.sql\" dist/actions && copyfiles -f \"migrations/actions/down/*.sql\" dist/actions/down", - "prune": "docker compose down && rm -rf volumes", - "setup": "pnpm db:start && pnpm db" + "db:generate": "schemats generate --config schemats.cjs -o src/db/types.ts" }, "dependencies": { "@sentry/node": "7.30.0", - "@slonik/migrator": "0.8.5", "@theguild/buddy": "0.1.0", "dotenv": "16.0.3", "got": "12.5.3", @@ -40,22 +27,8 @@ "@tgriesser/schemats": "7.0.1", "@types/node": "18.11.18", "@types/pg": "8.6.6", - "copyfiles": "2.4.1", "pg-promise": "10.15.4", - "ts-node": "10.9.1", "tslib": "2.4.1", "typescript": "4.9.4" - }, - "buildOptions": { - "runify": true, - "tsup": true, - "bin": "migrations/index.ts", - "external": [ - "pg-native" - ], - "tags": [ - "backend" - ], - "banner": "../../../scripts/banner.js" } } diff --git a/packages/services/storage/schemats.cjs b/packages/services/storage/schemats.cjs index b945c65b3..368fb5fb9 100644 --- a/packages/services/storage/schemats.cjs +++ b/packages/services/storage/schemats.cjs @@ -1,5 +1,5 @@ /* eslint-disable */ -const cn = require('./tools/db-connection-string.cjs'); +const cn = require('../../migrations/tools/db-connection-string.cjs'); module.exports = { conn: cn('registry'), diff --git a/packages/services/storage/tsconfig.json b/packages/services/storage/tsconfig.json index 3572993a8..9b376c2b1 100644 --- a/packages/services/storage/tsconfig.json +++ b/packages/services/storage/tsconfig.json @@ -1,9 +1,4 @@ { "extends": "../../../tsconfig.json", - "compilerOptions": { - "target": "ES2020", - "module": "esnext", - "rootDir": "../.." - }, - "include": ["src", "migrations"] + "include": ["src"] } diff --git a/patches/@slonik__migrator@0.8.5.patch b/patches/@slonik__migrator@0.8.5.patch new file mode 100644 index 000000000..73f3fd27a --- /dev/null +++ b/patches/@slonik__migrator@0.8.5.patch @@ -0,0 +1,44 @@ +diff --git a/dist/index.js b/dist/index.js +index 65ddf2942ab6f9ad969cdc5bbac36977c91fc984..df9cbe372f5c4034b51f8c22434778d87e2a152b 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -53,7 +53,7 @@ class SlonikMigrator extends umzug.Umzug { + } + /** Glob pattern with `migrationsPath` as `cwd`. Could be overridden to support nested directories */ + migrationsGlob() { +- return './*.{js,ts,sql}'; ++ return './*.{js,ts,mts,mjs,sql}'; + } + /** Gets a hexadecimal integer to pass to postgres's `select pg_advisory_lock()` function */ + advisoryLockId() { +@@ -66,10 +66,10 @@ class SlonikMigrator extends umzug.Umzug { + return slonik_1.sql.identifier(Array.isArray(table) ? table : [table]); + } + template(filepath) { +- if (filepath.endsWith('.ts')) { ++ if (filepath.endsWith('.ts') || filepath.endsWith('.mts')) { + return [[filepath, templates.typescript]]; + } +- if (filepath.endsWith('.js')) { ++ if (filepath.endsWith('.js') || filepath.endsWith('.mjs')) { + return [[filepath, templates.javascript]]; + } + const downPath = path.join(path.dirname(filepath), 'down', path.basename(filepath)); +@@ -93,12 +93,14 @@ class SlonikMigrator extends umzug.Umzug { + }; + } + const { connection: slonik } = params.context; +- const migrationModule = require(params.path); ++ // returns a Promise, but we'll await it later ++ const migrationModule = import(params.path); ++ + return { + name: params.name, + path: params.path, +- up: async (upParams) => migrationModule.up({ slonik, sql: slonik_1.sql, ...upParams }), +- down: async (downParams) => { var _a; return (_a = migrationModule.down) === null || _a === void 0 ? void 0 : _a.call(migrationModule, { slonik, sql: slonik_1.sql, ...downParams }); }, ++ up: async (upParams) => (await migrationModule).up({ slonik, sql: slonik_1.sql, ...upParams }), ++ down: async (downParams) => { var _a; return (_a = (await migrationModule).down) === null || _a === void 0 ? void 0 : _a.call(migrationModule, { slonik, sql: slonik_1.sql, ...downParams }); }, + }; + } + async getOrCreateMigrationsTable(context) { \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 00fe80781..63cfa5585 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,21 +1,12 @@ lockfileVersion: 5.4 patchedDependencies: - '@theguild/buddy@0.1.0': - hash: ryylgra5xglhidfoiaxehn22hq - path: patches/@theguild__buddy@0.1.0.patch - mjml-core@4.13.0: - hash: zxxsxbqejjmcwuzpigutzzq6wa - path: patches/mjml-core@4.13.0.patch - oclif@3.4.3: - hash: rxmtqiusuyruv7tkwux5gvotbm - path: patches/oclif@3.4.3.patch '@tgriesser/schemats@7.0.1': hash: u3kbucfchakklx3sci2vh6wjau path: patches/@tgriesser__schemats@7.0.1.patch - slonik@30.1.2: - hash: wg2hxbo7txnklmvja4aeqnygfi - path: patches/slonik@30.1.2.patch + '@theguild/buddy@0.1.0': + hash: ryylgra5xglhidfoiaxehn22hq + path: patches/@theguild__buddy@0.1.0.patch '@graphql-inspector/core@3.4.0': hash: tu4s2cjheabese6hdiqk5i3mii path: patches/@graphql-inspector__core@3.4.0.patch @@ -25,6 +16,18 @@ patchedDependencies: bullmq@3.5.6: hash: xbmkiyyfti7h6orsfs6pdmi4s4 path: patches/bullmq@3.5.6.patch + oclif@3.4.3: + hash: rxmtqiusuyruv7tkwux5gvotbm + path: patches/oclif@3.4.3.patch + mjml-core@4.13.0: + hash: zxxsxbqejjmcwuzpigutzzq6wa + path: patches/mjml-core@4.13.0.patch + '@slonik/migrator@0.8.5': + hash: yfpv2xzdmnkrtu2wvaktxf5mmq + path: patches/@slonik__migrator@0.8.5.patch + slonik@30.1.2: + hash: wg2hxbo7txnklmvja4aeqnygfi + path: patches/slonik@30.1.2.patch importers: @@ -271,7 +274,7 @@ importers: '@graphql-codegen/typescript-graphql-request': 4.5.8_upqs5jjd3ykece5dxcsvdam2ju '@types/env-ci': 3.1.1 '@types/mkdirp': 1.0.2 - oclif: 3.4.3 + oclif: 3.4.3_rxmtqiusuyruv7tkwux5gvotbm packages/libraries/client: specifiers: @@ -328,6 +331,35 @@ importers: graphql: 16.6.0 publishDirectory: dist + packages/migrations: + specifiers: + '@slonik/migrator': 0.8.5 + '@types/node': 18.11.18 + '@types/pg': 8.6.6 + copyfiles: 2.4.1 + dotenv: 16.0.3 + got: 12.5.3 + pg-promise: 10.15.4 + slonik: 30.1.2 + ts-node: 10.9.1 + tslib: 2.4.1 + typescript: 4.9.4 + zod: 3.20.2 + dependencies: + '@slonik/migrator': 0.8.5_yfpv2xzdmnkrtu2wvaktxf5mmq_slonik@30.1.2 + dotenv: 16.0.3 + got: 12.5.3 + slonik: 30.1.2_wg2hxbo7txnklmvja4aeqnygfi + ts-node: 10.9.1_awa2wsr5thmg3i7jqycphctjfq + zod: 3.20.2 + devDependencies: + '@types/node': 18.11.18 + '@types/pg': 8.6.6 + copyfiles: 2.4.1 + pg-promise: 10.15.4 + tslib: 2.4.1 + typescript: 4.9.4 + packages/services/api: specifiers: '@aws-sdk/client-s3': 3.245.0 @@ -701,12 +733,10 @@ importers: specifiers: '@sentry/node': 7.30.0 '@sentry/types': 7.30.0 - '@slonik/migrator': 0.8.5 '@tgriesser/schemats': 7.0.1 '@theguild/buddy': 0.1.0 '@types/node': 18.11.18 '@types/pg': 8.6.6 - copyfiles: 2.4.1 dotenv: 16.0.3 got: 12.5.3 param-case: 3.0.4 @@ -714,13 +744,11 @@ importers: slonik: 30.1.2 slonik-interceptor-query-logging: 1.4.7 slonik-utilities: 1.9.4 - ts-node: 10.9.1 tslib: 2.4.1 typescript: 4.9.4 zod: 3.20.2 dependencies: '@sentry/node': 7.30.0 - '@slonik/migrator': 0.8.5_slonik@30.1.2 '@theguild/buddy': 0.1.0_ryylgra5xglhidfoiaxehn22hq dotenv: 16.0.3 got: 12.5.3 @@ -734,9 +762,7 @@ importers: '@tgriesser/schemats': 7.0.1_u3kbucfchakklx3sci2vh6wjau '@types/node': 18.11.18 '@types/pg': 8.6.6 - copyfiles: 2.4.1 pg-promise: 10.15.4 - ts-node: 10.9.1_awa2wsr5thmg3i7jqycphctjfq tslib: 2.4.1 typescript: 4.9.4 @@ -10733,7 +10759,7 @@ packages: - debug dev: false - /@slonik/migrator/0.8.5_slonik@30.1.2: + /@slonik/migrator/0.8.5_yfpv2xzdmnkrtu2wvaktxf5mmq_slonik@30.1.2: resolution: {integrity: sha512-EqX2/n75oLPuf8E6595oxz4nsmEQ0AWpBPmbBjTmDS2AI0nXo5ED22objFGFHQ7Qy8k0Xja+BgDrIxgnHZNv7w==} peerDependencies: slonik: '>= 23.0.0 < 24' @@ -10741,6 +10767,7 @@ packages: slonik: 30.1.2_wg2hxbo7txnklmvja4aeqnygfi umzug: 3.0.0-beta.15 dev: false + patched: true /@stripe/react-stripe-js/1.16.3_keartrz7uyerl3wkpjpue4xbhy: resolution: {integrity: sha512-gS6UDGEuM92K50pFB3o//EqqPxmaqpC8IrsBda4P4LxeULoO0pBFSHXJ5Ab6uA7G2lyO2bluvSLereh0OH9GrQ==} @@ -15739,7 +15766,7 @@ packages: dependencies: fast-glob: 3.2.12 postcss: 8.4.20 - tailwindcss: 3.2.4_ts-node@10.9.1 + tailwindcss: 3.2.4_ra2vnoek4vhbzktaezawwqbin4 transitivePeerDependencies: - ts-node dev: true @@ -21863,7 +21890,7 @@ packages: resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} dev: false - /oclif/3.4.3: + /oclif/3.4.3_rxmtqiusuyruv7tkwux5gvotbm: resolution: {integrity: sha512-Kg8ezSJvDluazG6eyN+TeoryaFn3i9utKVty2hy6I0FUy6QcPnbl8+MCha4Jwc+x5+pA46Tu0n2cnjUcWqp56w==} engines: {node: '>=12.0.0'} hasBin: true @@ -21891,6 +21918,7 @@ packages: - encoding - supports-color dev: true + patched: true /on-exit-leak-free/2.1.0: resolution: {integrity: sha512-VuCaZZAjReZ3vUwgOB8LxAosIurDiAW0s13rI1YwmaP++jvcxP77AWoQvenZebpCA2m8WC1/EosPYPMjnRAp/w==} @@ -25636,10 +25664,12 @@ packages: - ts-node dev: true - /tailwindcss/3.2.4_ts-node@10.9.1: + /tailwindcss/3.2.4_ra2vnoek4vhbzktaezawwqbin4: resolution: {integrity: sha512-AhwtHCKMtR71JgeYDaswmZXhPcW9iuI9Sp2LvZPo9upDZ7231ZJ7eA9RaURbhpXGVlrjX4cFNlB4ieTetEb7hQ==} engines: {node: '>=12.13.0'} hasBin: true + peerDependencies: + postcss: ^8.0.9 dependencies: arg: 5.0.2 chokidar: 3.5.3 @@ -26068,7 +26098,7 @@ packages: typescript: 4.9.4 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - dev: true + dev: false /ts-node/10.9.1_wiov2xnxutssp33bmsvup4kx3q: resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} @@ -26202,7 +26232,7 @@ packages: smartwrap: 2.0.2 strip-ansi: 6.0.1 wcwidth: 1.0.1 - yargs: 17.6.0 + yargs: 17.6.2 dev: true /tunnel-agent/0.6.0: @@ -26382,7 +26412,6 @@ packages: resolution: {integrity: sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==} engines: {node: '>=4.2.0'} hasBin: true - dev: true /ua-parser-js/0.7.31: resolution: {integrity: sha512-qLK/Xe9E2uzmYI3qLeOmI0tEOt+TBBQyUIAh4aAgU05FVYzeZrKUdkAZfBNVGRaHVgV0TDkdEngJSw/SyQchkQ==} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0bebdd1bb..e9fa73603 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,6 @@ packages: - packages/services/* + - packages/migrations - packages/services/external-composition/* - packages/web/* - packages/libraries/*