Elgato_dark/console
1
0
Fork 0
mirror of https://github.com/graphql-hive/console synced 2026-04-21 14:37:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Upgrade Redis to v7 and upgrade BullMQ to latest (#937)

Browse source
This commit is contained in:
Dotan Simha 2023-01-02 13:05:23 +02:00 committed by GitHub
parent 032e86d123
commit 4d2ec4bc8a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 70 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
deployment/services/emails.ts
View file

@ -43,7 +43,6 @@ export function deployEmails({
REDIS_HOST: redis.config.host,
REDIS_PORT: String(redis.config.port),
REDIS_PASSWORD: redis.config.password,
BULLMQ_COMMANDS_FROM_ROOT: 'true',
EMAIL_FROM: email.from,
EMAIL_PROVIDER: 'postmark',
EMAIL_PROVIDER_POSTMARK_TOKEN: email.token,

1
deployment/services/webhooks.ts
View file

@ -40,7 +40,6 @@ export function deployWebhooks({
REDIS_HOST: redis.config.host,
REDIS_PORT: String(redis.config.port),
REDIS_PASSWORD: redis.config.password,
BULLMQ_COMMANDS_FROM_ROOT: 'true',
REQUEST_BROKER: '1',
REQUEST_BROKER_ENDPOINT: broker.workerBaseUrl,
REQUEST_BROKER_SIGNATURE: broker.secretSignature,

8
deployment/utils/redis.ts
View file

@ -1,8 +1,8 @@
import * as k8s from '@pulumi/kubernetes';
import * as kx from '@pulumi/kubernetesx';
import { getLocalComposeConfig } from './local-config';
import { normalizeEnv, PodBuilder } from './pod-builder';
const DEFAULT_IMAGE = 'bitnami/redis:6.2.6';
const PORT = 6379;
export class Redis {
@ -14,8 +14,8 @@ export class Redis {
) {}
deploy({ limits }: { limits: k8s.types.input.core.v1.ResourceRequirements['limits'] }) {
const redisService = getLocalComposeConfig().service('redis');
const name = 'redis-store';
const image = DEFAULT_IMAGE;
const env = normalizeEnv(this.options.env ?? {}).concat([
{
@ -58,10 +58,10 @@ fi
containers: [
{
name,
image,
image: redisService.image,
env,
volumeMounts,
ports: [{ containerPort: PORT, hostPort: PORT, protocol: 'TCP' }],
ports: [{ containerPort: PORT, protocol: 'TCP' }],
resources: {
limits,
},

4
docker-compose.community.yml
View file

@ -96,7 +96,7 @@ services:
- ./.hive/broker/db:/var/lib/kafka/data
redis:
image: bitnami/redis:6.2
image: bitnami/redis:7.0.7
networks:
- 'stack'
healthcheck:
@ -291,7 +291,6 @@ services:
condition: service_healthy
environment:
NODE_ENV: production
BULLMQ_COMMANDS_FROM_ROOT: 'true'
PORT: 3005
REDIS_HOST: redis
REDIS_PORT: 6379
@ -306,7 +305,6 @@ services:
condition: service_healthy
environment:
NODE_ENV: production
BULLMQ_COMMANDS_FROM_ROOT: 'true'
PORT: 3011
REDIS_HOST: redis
REDIS_PORT: 6379

8
docs/TESTING.md
View file

@ -109,7 +109,13 @@ To run integration tests locally, from the pre-build Docker image, follow:
2. Install all deps: `pnpm i`
3. Generate types: `pnpm graphql:generate`
4. Build source code: `pnpm build`
5. Decide on the commit ID / Docker image tag you would like to use.
5. Decide on the commit ID / Docker image tag you would like to use and set it as env var:
```
export DOCKER_REGISTRY="ghcr.io/kamilkisiela/graphql-hive/"
export DOCKER_TAG=":IMAGE_TAG_HERE"
```
6. Run the e2e environment, by running:
`docker compose -f docker-compose.community.yml --env-file ./integration-tests/.env up -d --wait`
7. Run Cypress: `pnpm test:e2e`

1
integration-tests/docker-compose.integration.yaml
View file

@ -147,7 +147,6 @@ services:
POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
USAGE_ESTIMATOR_ENDPOINT: http://usage-estimator:3008
EMAILS_ENDPOINT: http://emails:3011
BULLMQ_COMMANDS_FROM_ROOT: 'true'
PORT: 3009
# Overrides only to `docker-compose.community.yaml` from now on:

4
package.json
View file

@ -112,11 +112,11 @@
"patchedDependencies": {
"@tgriesser/schemats@7.0.1": "patches/@tgriesser__schemats@7.0.1.patch",
"@theguild/buddy@0.1.0": "patches/@theguild__buddy@0.1.0.patch",
"bullmq@1.81.4": "patches/bullmq@1.81.4.patch",
"mjml-core@4.13.0": "patches/mjml-core@4.13.0.patch",
"slonik@30.1.2": "patches/slonik@30.1.2.patch",
"@graphql-inspector/core@3.4.0": "patches/@graphql-inspector__core@3.4.0.patch",
"@oclif/core@1.23.0": "patches/@oclif__core@1.23.0.patch"
"@oclif/core@1.23.0": "patches/@oclif__core@1.23.0.patch",
"bullmq@3.5.1": "patches/bullmq@3.5.1.patch"
}
}
}

2
packages/services/emails/package.json
View file

@ -14,7 +14,7 @@
"@sentry/node": "7.28.1",
"@trpc/server": "10.7.0",
"@whatwg-node/fetch": "0.5.3",
"bullmq": "1.81.4",
"bullmq": "3.5.1",
"dotenv": "16.0.3",
"ioredis": "5.2.4",
"mjml": "4.13.0",

29
packages/services/emails/src/scheduler.ts
View file

@ -1,6 +1,6 @@
import type { FastifyLoggerInstance } from '@hive/service-common';
import * as Sentry from '@sentry/node';
import { Job, Queue, QueueScheduler, Worker } from 'bullmq';
import { Job, Queue, Worker } from 'bullmq';
import Redis, { Redis as RedisInstance } from 'ioredis';
import mjml2html from 'mjml';
import pTimeout from 'p-timeout';
@ -24,7 +24,6 @@ export function createScheduler(config: {
}) {
let redisConnection: RedisInstance | null;
let queue: Queue | null;
let queueScheduler: QueueScheduler | null;
let stopped = false;
const logger = config.logger;
@ -41,12 +40,12 @@ export function createScheduler(config: {
};
}
function onFailed(job: Job, error: Error) {
function onFailed(job: Job<EmailInput> | undefined, error: Error) {
logger.debug(
`Job %s failed after %s attempts, reason: %s`,
job.name,
job.attemptsMade,
job.failedReason,
job?.name,
job?.attemptsMade,
job?.failedReason,
);
logger.error(error);
emailsFailuresTotal.inc();
@ -59,20 +58,14 @@ export function createScheduler(config: {
const prefix = 'hive-emails';
queueScheduler = new QueueScheduler(config.queueName, {
prefix,
connection: redisConnection,
sharedConnection: true,
});
queue = new Queue(config.queueName, {
prefix,
connection: redisConnection,
sharedConnection: true,
});
// Wait for Queues and Scheduler to be ready
await Promise.all([queueScheduler.waitUntilReady(), queue.waitUntilReady()]);
// Wait for Queues to be ready
await queue.waitUntilReady();
const worker = new Worker<EmailInput>(
config.queueName,
@ -171,17 +164,17 @@ export function createScheduler(config: {
logger.info('Clearing BullMQ...');
try {
queue?.removeAllListeners();
queueScheduler?.removeAllListeners(),
await pTimeout(Promise.all([queue?.close(), queueScheduler?.close()]), {
if (queue) {
queue.removeAllListeners();
await pTimeout(queue.close(), {
milliseconds: 5000,
message: 'BullMQ close timeout',
});
}
} catch (e) {
logger.error('Failed to stop queues', e);
} finally {
queue = null;
queueScheduler = null;
logger.info('BullMQ stopped');
}

2
packages/services/storage/docker-compose.yml
View file

@ -20,7 +20,7 @@ services:
- '5432:5432'
redis:
image: docker.io/bitnami/redis:6.2
image: docker.io/bitnami/redis:7.0.7
networks:
- 'stack'
healthcheck:

2
packages/services/webhooks/package.json
View file

@ -14,7 +14,7 @@
"@sentry/node": "7.28.1",
"@sentry/tracing": "7.28.1",
"@trpc/server": "10.7.0",
"bullmq": "1.81.4",
"bullmq": "3.5.1",
"dotenv": "16.0.3",
"got": "12.5.3",
"ioredis": "5.2.4",

37
packages/services/webhooks/src/scheduler.ts
View file

@ -1,5 +1,5 @@
import * as Sentry from '@sentry/node';
import { Job, Queue, QueueScheduler, Worker } from 'bullmq';
import { Job, Queue, Worker } from 'bullmq';
import Redis, { Redis as RedisInstance } from 'ioredis';
import pTimeout from 'p-timeout';
import { createWebhookJob, scheduleWebhook } from './jobs';
@ -38,7 +38,6 @@ export interface WebhookInput {
export function createScheduler(config: Config) {
let redisConnection: RedisInstance | null;
let webhookQueue: Queue | null;
let webhookQueueScheduler: QueueScheduler | null;
let stopped = false;
const logger = config.logger;
@ -46,17 +45,17 @@ export function createScheduler(config: Config) {
logger.info('Clearing BullMQ...');
try {
webhookQueue?.removeAllListeners();
webhookQueueScheduler?.removeAllListeners(),
await pTimeout(Promise.all([webhookQueue?.close(), webhookQueueScheduler?.close()]), {
if (webhookQueue) {
webhookQueue.removeAllListeners();
await pTimeout(webhookQueue.close(), {
milliseconds: 5000,
message: 'BullMQ close timeout',
});
}
} catch (e) {
logger.error('Failed to stop queues', e);
} finally {
webhookQueue = null;
webhookQueueScheduler = null;
logger.info('BullMQ stopped');
}
}
@ -68,20 +67,14 @@ export function createScheduler(config: Config) {
const prefix = 'hive-webhooks';
webhookQueueScheduler = new QueueScheduler(config.webhookQueueName, {
prefix,
connection: redisConnection,
sharedConnection: true,
});
webhookQueue = new Queue(config.webhookQueueName, {
prefix,
connection: redisConnection,
sharedConnection: true,
});
// Wait for Queues and Scheduler to be ready
await Promise.all([webhookQueueScheduler.waitUntilReady(), webhookQueue.waitUntilReady()]);
// Wait for Queue to be ready
await webhookQueue.waitUntilReady();
const webhookJob = createWebhookJob({ config });
@ -158,16 +151,16 @@ export function createScheduler(config: Config) {
};
}
function onFailed(job: Job, error: Error) {
function onFailed(job: Job | undefined, error: Error) {
logger.debug(
`Job %s failed after %s attempts, reason: %s (orgId=%s, projectId=%s, targetId=%s, schemaId=%s)`,
job.name,
job.attemptsMade,
job.failedReason,
job.data?.event?.organization?.id,
job.data?.event?.project?.id,
job.data?.event?.target?.id,
job.data?.event?.schema?.id,
job?.name,
job?.attemptsMade,
job?.failedReason,
job?.data?.event?.organization?.id,
job?.data?.event?.project?.id,
job?.data?.event?.target?.id,
job?.data?.event?.schema?.id,
);
logger.error(error);
}

21
patches/bullmq@1.81.4.patch → patches/bullmq@3.5.1.patch
View file

@ -1,18 +1,5 @@
diff --git a/dist/esm/classes/redis-connection.js b/dist/esm/classes/redis-connection.js
index 7843e65..3807efe 100644
--- a/dist/esm/classes/redis-connection.js
+++ b/dist/esm/classes/redis-connection.js
@@ -105,7 +105,7 @@ export class RedisConnection extends EventEmitter {
}
loadCommands() {
return (this._client['bullmq:loadingCommands'] ||
- (this._client['bullmq:loadingCommands'] = scriptLoader.load(this._client, path.join(__dirname, '../commands'))));
+ (this._client['bullmq:loadingCommands'] = scriptLoader.load(this._client, path.join(__dirname, process.env.BULLMQ_COMMANDS_FROM_ROOT ? './' : '../commands'))));
}
async init() {
if (!this._client) {
diff --git a/dist/esm/commands/script-loader.js b/dist/esm/commands/script-loader.js
index 3ed1c00..79e455a 100644
index bcbe1b33d8c4adf3113e561584dbd17c48a197d2..671d1316123c29ea8bc0901e4e8236fbd6290dcd 100644
--- a/dist/esm/commands/script-loader.js
+++ b/dist/esm/commands/script-loader.js
@@ -2,6 +2,7 @@ import { createHash } from 'crypto';
@ -23,11 +10,11 @@ index 3ed1c00..79e455a 100644
import { promisify } from 'util';
const readFile = promisify(fs.readFile);
const readdir = promisify(fs.readdir);
@@ -343,15 +344,7 @@ async function getFilenamesByPattern(pattern) {
@@ -349,15 +350,7 @@ async function getFilenamesByPattern(pattern) {
// Determine the project root
// https://stackoverflow.com/a/18721515
function getPkgJsonDir() {
- for (const modPath of (module.paths || [])) {
- for (const modPath of module.paths || []) {
- try {
- const prospectivePkgJsonDir = path.dirname(modPath);
- fs.accessSync(modPath, fs.constants.F_OK);
@ -39,4 +26,4 @@ index 3ed1c00..79e455a 100644
+ return path.dirname(fileURLToPath(import.meta.url));
}
// https://stackoverflow.com/a/66842927
// some dark magic here :-)
// some dark magic here :-)

86
pnpm-lock.yaml
View file

@ -1,27 +1,27 @@
lockfileVersion: 5.4
patchedDependencies:
'@tgriesser/schemats@7.0.1':
hash: u3kbucfchakklx3sci2vh6wjau
path: patches/@tgriesser__schemats@7.0.1.patch
bullmq@1.81.4:
hash: d7likfulu5mfjaaerbpls6og24
path: patches/bullmq@1.81.4.patch
'@theguild/buddy@0.1.0':
hash: ryylgra5xglhidfoiaxehn22hq
path: patches/@theguild__buddy@0.1.0.patch
'@oclif/core@1.23.0':
hash: zhte2hlj7lfobytjpalcwwo474
path: patches/@oclif__core@1.23.0.patch
mjml-core@4.13.0:
hash: zxxsxbqejjmcwuzpigutzzq6wa
path: patches/mjml-core@4.13.0.patch
'@tgriesser/schemats@7.0.1':
hash: u3kbucfchakklx3sci2vh6wjau
path: patches/@tgriesser__schemats@7.0.1.patch
slonik@30.1.2:
hash: wg2hxbo7txnklmvja4aeqnygfi
path: patches/slonik@30.1.2.patch
'@graphql-inspector/core@3.4.0':
hash: tu4s2cjheabese6hdiqk5i3mii
path: patches/@graphql-inspector__core@3.4.0.patch
'@oclif/core@1.23.0':
hash: zhte2hlj7lfobytjpalcwwo474
path: patches/@oclif__core@1.23.0.patch
bullmq@3.5.1:
hash: xbmkiyyfti7h6orsfs6pdmi4s4
path: patches/bullmq@3.5.1.patch
importers:
@ -459,7 +459,7 @@ importers:
'@types/nodemailer': 6.4.7
'@types/sendmail': 1.4.4
'@whatwg-node/fetch': 0.5.3
bullmq: 1.81.4
bullmq: 3.5.1
copyfiles: 2.4.1
dotenv: 16.0.3
ioredis: 5.2.4
@ -474,7 +474,7 @@ importers:
'@sentry/node': 7.28.1
'@trpc/server': 10.7.0
'@whatwg-node/fetch': 0.5.3
bullmq: 1.81.4_d7likfulu5mfjaaerbpls6og24
bullmq: 3.5.1
dotenv: 16.0.3
ioredis: 5.2.4
mjml: 4.13.0
@ -910,7 +910,7 @@ importers:
'@sentry/node': 7.28.1
'@sentry/tracing': 7.28.1
'@trpc/server': 10.7.0
bullmq: 1.81.4
bullmq: 3.5.1
copyfiles: 2.4.1
dotenv: 16.0.3
got: 12.5.3
@ -923,7 +923,7 @@ importers:
'@sentry/node': 7.28.1
'@sentry/tracing': 7.28.1
'@trpc/server': 10.7.0
bullmq: 1.81.4_d7likfulu5mfjaaerbpls6og24
bullmq: 3.5.1
dotenv: 16.0.3
got: 12.5.3
ioredis: 5.2.4
@ -12530,22 +12530,20 @@ packages:
semver: 7.3.8
dev: true
/bullmq/1.81.4_d7likfulu5mfjaaerbpls6og24:
resolution: {integrity: sha512-sUEWOMKZnWlh1/XNqYAoSwXW6P8nZN7uJiHKZ8XlZCiIxWlEGjFtlugkkiCZ0lsTI2nNRHdxfpn78x9K3L1utQ==}
/bullmq/3.5.1:
resolution: {integrity: sha512-QxQnSIPBmEVq/bM1wxmMdh9RS4oNyUNeCvoTegr2EgtlNYToCkLgHAkM56JhaRlffo9ExNiWfVFJpXLWDfK0Fw==}
dependencies:
cron-parser: 4.6.0
get-port: 5.1.1
glob: 7.2.3
ioredis: 4.28.5
glob: 8.0.3
ioredis: 5.2.4
lodash: 4.17.21
msgpackr: 1.7.2
semver: 6.3.0
tslib: 1.14.1
uuid: 8.3.2
semver: 7.3.8
tslib: 2.4.1
uuid: 9.0.0
transitivePeerDependencies:
- supports-color
dev: false
patched: true
/bundle-require/3.1.2_esbuild@0.15.15:
resolution: {integrity: sha512-Of6l6JBAxiyQ5axFxUM6dYeP/W7X2Sozeo/4EYB9sJhL+dqL7TKjg+shwxp6jlu/6ZSERfsYtIpSJ1/x3XkAEA==}
@ -14458,11 +14456,6 @@ packages:
/delegates/1.0.0:
resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==}
/denque/1.5.1:
resolution: {integrity: sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==}
engines: {node: '>=0.10'}
dev: false
/denque/2.1.0:
resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
engines: {node: '>=0.10'}
@ -15436,7 +15429,7 @@ packages:
dependencies:
fast-glob: 3.2.12
postcss: 8.4.20
tailwindcss: 3.2.4_ra2vnoek4vhbzktaezawwqbin4
tailwindcss: 3.2.4_ts-node@10.9.1
transitivePeerDependencies:
- ts-node
dev: true
@ -16427,11 +16420,6 @@ packages:
resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==}
engines: {node: '>=8.0.0'}
/get-port/5.1.1:
resolution: {integrity: sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==}
engines: {node: '>=8'}
dev: false
/get-stack-trace/2.1.1:
resolution: {integrity: sha512-dhqSDD9lHU/6FvIZ9KbXGmVK6IKr9ZskZtNOUvhlCiONlnqatu4FmAeRbxCfJJVuQ0NWfz6dAbibKQg19B7AmQ==}
engines: {node: '>=8.0'}
@ -17535,25 +17523,6 @@ packages:
ioredis: 5.2.4
dev: true
/ioredis/4.28.5:
resolution: {integrity: sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==}
engines: {node: '>=6'}
dependencies:
cluster-key-slot: 1.1.1
debug: 4.3.4
denque: 1.5.1
lodash.defaults: 4.2.0
lodash.flatten: 4.4.0
lodash.isarguments: 3.1.0
p-map: 2.1.0
redis-commands: 1.7.0
redis-errors: 1.2.0
redis-parser: 3.0.0
standard-as-callback: 2.1.0
transitivePeerDependencies:
- supports-color
dev: false
/ioredis/5.2.4:
resolution: {integrity: sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==}
engines: {node: '>=12.22.0'}
@ -19217,10 +19186,6 @@ packages:
resolution: {integrity: sha512-/OcpcAGWlrZyoHGeHh3cAoa6nGdX6QYtmzNP84Jqol6UEQQ2gIaU3H+0eICcjcKGl0/XF8LWOujNn9lffsnaOg==}
dev: false
/lodash.flatten/4.4.0:
resolution: {integrity: sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==}
dev: false
/lodash.get/4.4.2:
resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==}
@ -21548,6 +21513,7 @@ packages:
/p-map/2.1.0:
resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==}
engines: {node: '>=6'}
dev: true
/p-map/4.0.0:
resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==}
@ -23435,10 +23401,6 @@ packages:
dependencies:
esprima: 4.0.1
/redis-commands/1.7.0:
resolution: {integrity: sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==}
dev: false
/redis-errors/1.2.0:
resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
engines: {node: '>=4'}
@ -25064,12 +25026,10 @@ packages:
- ts-node
dev: true
/tailwindcss/3.2.4_ra2vnoek4vhbzktaezawwqbin4:
/tailwindcss/3.2.4_ts-node@10.9.1:
resolution: {integrity: sha512-AhwtHCKMtR71JgeYDaswmZXhPcW9iuI9Sp2LvZPo9upDZ7231ZJ7eA9RaURbhpXGVlrjX4cFNlB4ieTetEb7hQ==}
engines: {node: '>=12.13.0'}
hasBin: true
peerDependencies:
postcss: ^8.0.9
dependencies:
arg: 5.0.2
chokidar: 3.5.3