From df0310dce634fa46e646fbb46f0800b807f5cf08 Mon Sep 17 00:00:00 2001 From: Dotan Simha Date: Mon, 4 Mar 2024 14:56:12 +0200 Subject: [PATCH] Refactor deployment code (#4138) --- deployment/index.ts | 360 ++++++++++-------------- deployment/services/app.ts | 247 ++++++---------- deployment/services/billing.ts | 81 +++--- deployment/services/cf-broker.ts | 50 ++-- deployment/services/cf-cdn.ts | 60 ++-- deployment/services/clickhouse.ts | 28 +- deployment/services/database-cleanup.ts | 7 +- deployment/services/db-migrations.ts | 81 +++--- deployment/services/docker.ts | 21 ++ deployment/services/emails.ts | 67 +++-- deployment/services/environment.ts | 45 +++ deployment/services/github.ts | 22 ++ deployment/services/graphql.ts | 267 ++++++++---------- deployment/services/kafka.ts | 27 +- deployment/services/observability.ts | 9 +- deployment/services/policy.ts | 36 ++- deployment/services/postgres.ts | 39 +++ deployment/services/proxy.ts | 17 +- deployment/services/rate-limit.ts | 63 ++--- deployment/services/redis.ts | 30 +- deployment/services/s3.ts | 24 ++ deployment/services/schema.ts | 47 ++-- deployment/services/sentry-events.ts | 23 +- deployment/services/sentry.ts | 29 ++ deployment/services/slack-app.ts | 22 ++ deployment/services/supertokens.ts | 45 ++- deployment/services/tokens.ts | 68 ++--- deployment/services/usage-estimation.ts | 46 ++- deployment/services/usage-ingestor.ts | 78 +++-- deployment/services/usage.ts | 51 ++-- deployment/services/webhooks.ts | 52 ++-- deployment/services/zendesk.ts | 33 +++ deployment/types.ts | 6 - deployment/utils/cloudflare.ts | 20 +- deployment/utils/helpers.ts | 18 -- deployment/utils/redis.ts | 39 +-- deployment/utils/secrets.ts | 21 ++ deployment/utils/service-deployment.ts | 50 +++- 38 files changed, 1184 insertions(+), 1045 deletions(-) create mode 100644 deployment/services/docker.ts create mode 100644 deployment/services/environment.ts create mode 100644 deployment/services/github.ts create mode 100644 deployment/services/postgres.ts create mode 100644 deployment/services/s3.ts create mode 100644 deployment/services/sentry.ts create mode 100644 deployment/services/slack-app.ts create mode 100644 deployment/services/zendesk.ts create mode 100644 deployment/utils/secrets.ts diff --git a/deployment/index.ts b/deployment/index.ts index f7f0dfb26..4d1eea6a4 100644 --- a/deployment/index.ts +++ b/deployment/index.ts @@ -1,5 +1,4 @@ import * as pulumi from '@pulumi/pulumi'; -import * as random from '@pulumi/random'; import { deployApp } from './services/app'; import { deployStripeBilling } from './services/billing'; import { deployCFBroker } from './services/cf-broker'; @@ -8,41 +7,32 @@ import { deployClickhouse } from './services/clickhouse'; import { deployCloudFlareSecurityTransform } from './services/cloudflare-security'; import { deployDatabaseCleanupJob } from './services/database-cleanup'; import { deployDbMigrations } from './services/db-migrations'; +import { configureDocker } from './services/docker'; import { deployEmails } from './services/emails'; +import { prepareEnvironment } from './services/environment'; +import { configureGithubApp } from './services/github'; import { deployGraphQL } from './services/graphql'; import { deployKafka } from './services/kafka'; import { deployMetrics } from './services/observability'; import { deploySchemaPolicy } from './services/policy'; +import { deployPostgres } from './services/postgres'; import { deployProxy } from './services/proxy'; import { deployRateLimit } from './services/rate-limit'; import { deployRedis } from './services/redis'; +import { deployS3 } from './services/s3'; import { deploySchema } from './services/schema'; +import { configureSentry } from './services/sentry'; import { deploySentryEventsMonitor } from './services/sentry-events'; +import { configureSlackApp } from './services/slack-app'; import { deploySuperTokens } from './services/supertokens'; import { deployTokens } from './services/tokens'; import { deployUsage } from './services/usage'; import { deployUsageEstimation } from './services/usage-estimation'; import { deployUsageIngestor } from './services/usage-ingestor'; import { deployWebhooks } from './services/webhooks'; -import { DeploymentEnvironment } from './types'; +import { configureZendesk } from './services/zendesk'; import { optimizeAzureCluster } from './utils/azure-helpers'; -import { createDockerImageFactory } from './utils/docker-images'; -import { isDefined, isProduction } from './utils/helpers'; - -// eslint-disable-next-line no-process-env -process.env.PULUMI_K8S_SUPPRESS_HELM_HOOK_WARNINGS = '1'; - -optimizeAzureCluster(); - -const dockerConfig = new pulumi.Config('docker'); -const dockerImages = createDockerImageFactory({ - registryHostname: dockerConfig.require('registryUrl'), - imagesPrefix: dockerConfig.require('imagesPrefix'), -}); - -const imagePullSecret = dockerImages.createRepositorySecret( - dockerConfig.requireSecret('registryAuthBase64'), -); +import { isDefined } from './utils/helpers'; // eslint-disable-next-line no-process-env const imagesTag = process.env.DOCKER_IMAGE_TAG as string; @@ -51,250 +41,202 @@ if (!imagesTag) { throw new Error(`DOCKER_IMAGE_TAG env variable is not set.`); } +optimizeAzureCluster(); + +const docker = configureDocker(); const envName = pulumi.getStack(); -const commonConfig = new pulumi.Config('common'); -const appDns = 'app'; -const rootDns = commonConfig.require('dnsZone'); -const appHostname = `${appDns}.${rootDns}`; - const heartbeatsConfig = new pulumi.Config('heartbeats'); -const emailConfig = new pulumi.Config('email'); -const r2Config = new pulumi.Config('r2'); -const s3Config = { - endpoint: r2Config.require('endpoint'), - bucketName: r2Config.require('bucketName'), - accessKeyId: r2Config.requireSecret('accessKeyId'), - secretAccessKey: r2Config.requireSecret('secretAccessKey'), -}; - -const deploymentEnv: DeploymentEnvironment = { - ENVIRONMENT: envName, - NODE_ENV: 'production', - DEPLOYED_DNS: appHostname, -}; - -deploySentryEventsMonitor({ envName, imagePullSecret }); +const sentry = configureSentry(); +const environment = prepareEnvironment({ + release: imagesTag, + environment: envName, + rootDns: new pulumi.Config('common').require('dnsZone'), +}); +deploySentryEventsMonitor({ docker, environment, sentry }); deployMetrics({ envName }); - -const cdnAuthPrivateKey = commonConfig.requireSecret('cdnAuthPrivateKey'); +const clickhouse = deployClickhouse(); +const postgres = deployPostgres(); +const redis = deployRedis({ environment }); +const kafka = deployKafka(); +const s3 = deployS3(); const cdn = deployCFCDN({ - envName, - rootDns, - s3Config, - release: imagesTag, + s3, + sentry, + environment, }); -const cfBroker = deployCFBroker({ - envName, - rootDns, - release: imagesTag, +const broker = deployCFBroker({ + environment, + sentry, }); -const redisApi = deployRedis({ deploymentEnv }); -const kafkaApi = deployKafka(); -const clickhouseApi = deployClickhouse(); - // eslint-disable-next-line no-process-env const shouldCleanDatabase = process.env.CLEAN_DATABASE === 'true'; -const databaseCleanupJob = shouldCleanDatabase ? deployDatabaseCleanupJob({ deploymentEnv }) : null; +const databaseCleanupJob = shouldCleanDatabase ? deployDatabaseCleanupJob({ environment }) : null; // eslint-disable-next-line no-process-env const forceRunDbMigrations = process.env.FORCE_DB_MIGRATIONS === 'true'; const dbMigrations = deployDbMigrations({ - clickhouse: clickhouseApi, - kafka: kafkaApi, - deploymentEnv, - image: dockerImages.getImageId('storage', imagesTag), - imagePullSecret, + clickhouse, + docker, + postgres, + s3, + cdn, + environment, + image: docker.factory.getImageId('storage', imagesTag), force: forceRunDbMigrations, dependencies: [databaseCleanupJob].filter(isDefined), - s3: s3Config, - cdnAuthPrivateKey, }); -const tokensApi = deployTokens({ - image: dockerImages.getImageId('tokens', imagesTag), - release: imagesTag, - deploymentEnv, +const tokens = deployTokens({ + image: docker.factory.getImageId('tokens', imagesTag), + environment, dbMigrations, - redis: redisApi, + docker, + postgres, + redis, heartbeat: heartbeatsConfig.get('tokens'), - imagePullSecret, + sentry, }); -const webhooksApi = deployWebhooks({ - image: dockerImages.getImageId('webhooks', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, - redis: redisApi, +const webhooks = deployWebhooks({ + image: docker.factory.getImageId('webhooks', imagesTag), + environment, heartbeat: heartbeatsConfig.get('webhooks'), - broker: cfBroker, + broker, + docker, + redis, + sentry, }); -const emailsApi = deployEmails({ - image: dockerImages.getImageId('emails', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, - redis: redisApi, - email: { - token: emailConfig.requireSecret('token'), - from: emailConfig.require('from'), - messageStream: emailConfig.require('messageStream'), - }, - // heartbeat: heartbeatsConfig.get('emails'), +const emails = deployEmails({ + image: docker.factory.getImageId('emails', imagesTag), + docker, + environment, + redis, + sentry, }); -const usageEstimationApi = deployUsageEstimation({ - image: dockerImages.getImageId('usage-estimator', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, - clickhouse: clickhouseApi, +const usageEstimator = deployUsageEstimation({ + image: docker.factory.getImageId('usage-estimator', imagesTag), + docker, + environment, + clickhouse, dbMigrations, + sentry, }); -const billingApi = deployStripeBilling({ - image: dockerImages.getImageId('stripe-billing', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, +const billing = deployStripeBilling({ + image: docker.factory.getImageId('stripe-billing', imagesTag), + docker, + postgres, + environment, dbMigrations, - usageEstimator: usageEstimationApi, + usageEstimator, + sentry, }); -const rateLimitApi = deployRateLimit({ - image: dockerImages.getImageId('rate-limit', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, +const rateLimit = deployRateLimit({ + image: docker.factory.getImageId('rate-limit', imagesTag), + docker, + environment, dbMigrations, - usageEstimator: usageEstimationApi, - emails: emailsApi, + usageEstimator, + emails, + postgres, + sentry, }); -const usageApi = deployUsage({ - image: dockerImages.getImageId('usage', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, - tokens: tokensApi, - kafka: kafkaApi, +const usage = deployUsage({ + image: docker.factory.getImageId('usage', imagesTag), + docker, + environment, + tokens, + kafka, dbMigrations, - rateLimit: rateLimitApi, + rateLimit, + sentry, }); -const usageIngestorApi = deployUsageIngestor({ - image: dockerImages.getImageId('usage-ingestor', imagesTag), - imagePullSecret, - release: imagesTag, - clickhouse: clickhouseApi, - kafka: kafkaApi, - deploymentEnv, +const usageIngestor = deployUsageIngestor({ + image: docker.factory.getImageId('usage-ingestor', imagesTag), + docker, + clickhouse, + kafka, + environment, dbMigrations, heartbeat: heartbeatsConfig.get('usageIngestor'), + sentry, }); -const schemaApi = deploySchema({ - image: dockerImages.getImageId('schema', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, - redis: redisApi, - broker: cfBroker, +const schema = deploySchema({ + image: docker.factory.getImageId('schema', imagesTag), + docker, + environment, + redis, + broker, + sentry, }); -const schemaPolicyApi = deploySchemaPolicy({ - image: dockerImages.getImageId('policy', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, +const schemaPolicy = deploySchemaPolicy({ + image: docker.factory.getImageId('policy', imagesTag), + docker, + environment, + sentry, }); -const supertokensApiKey = new random.RandomPassword('supertokens-api-key', { - length: 31, - special: false, -}); +const supertokens = deploySuperTokens(postgres, { dependencies: [dbMigrations] }, environment); +const zendesk = configureZendesk({ environment }); +const githubApp = configureGithubApp(); +const slackApp = configureSlackApp(); -const oauthConfig = new pulumi.Config('oauth'); - -const githubConfig = { - clientId: oauthConfig.requireSecret('githubClient'), - clientSecret: oauthConfig.requireSecret('githubSecret'), -}; - -const googleConfig = { - clientId: oauthConfig.requireSecret('googleClient'), - clientSecret: oauthConfig.requireSecret('googleSecret'), -}; - -const supertokens = deploySuperTokens( - { apiKey: supertokensApiKey.result }, - { dependencies: [dbMigrations] }, - deploymentEnv, -); - -const zendeskConfig = new pulumi.Config('zendesk'); - -const graphqlApi = deployGraphQL({ - clickhouse: clickhouseApi, - image: dockerImages.getImageId('server', imagesTag), - imagePullSecret, - release: imagesTag, - deploymentEnv, - tokens: tokensApi, - webhooks: webhooksApi, - schema: schemaApi, - schemaPolicy: schemaPolicyApi, +const graphql = deployGraphQL({ + postgres, + environment, + clickhouse, + image: docker.factory.getImageId('server', imagesTag), + docker, + tokens, + webhooks, + schema, + schemaPolicy, dbMigrations, - redis: redisApi, - usage: usageApi, - cdnAuthPrivateKey, + redis, + usage, cdn, - usageEstimator: usageEstimationApi, - rateLimit: rateLimitApi, - billing: billingApi, - emails: emailsApi, - supertokensConfig: { - apiKey: supertokensApiKey.result, - endpoint: supertokens.localEndpoint, - }, - s3Config, - zendeskConfig: isProduction(deploymentEnv) - ? { - subdomain: zendeskConfig.require('subdomain'), - username: zendeskConfig.require('username'), - password: zendeskConfig.requireSecret('password'), - } - : null, + usageEstimator, + rateLimit, + billing, + emails, + supertokens, + s3, + zendesk, + githubApp, + sentry, }); const app = deployApp({ - deploymentEnv, - graphql: graphqlApi, + environment, + graphql, dbMigrations, - image: dockerImages.getImageId('app', imagesTag), - imagePullSecret, - release: imagesTag, - supertokensConfig: { - apiKey: supertokensApiKey.result, - endpoint: supertokens.localEndpoint, - }, - githubConfig, - googleConfig, - emailsEndpoint: emailsApi.localEndpoint, - zendeskSupport: isProduction(deploymentEnv), + image: docker.factory.getImageId('app', imagesTag), + docker, + supertokens, + emails, + zendesk, + billing, + github: githubApp, + slackApp, + sentry, }); const proxy = deployProxy({ - appHostname, app, - graphql: graphqlApi, - usage: usageApi, - deploymentEnv, + graphql, + usage, + environment, }); deployCloudFlareSecurityTransform({ @@ -315,12 +257,12 @@ deployCloudFlareSecurityTransform({ ignoredHosts: ['cdn.graphql-hive.com', 'cdn.staging.graphql-hive.com'], }); -export const graphqlApiServiceId = graphqlApi.service.id; -export const usageApiServiceId = usageApi.service.id; -export const usageIngestorApiServiceId = usageIngestorApi.service.id; -export const tokensApiServiceId = tokensApi.service.id; -export const schemaApiServiceId = schemaApi.service.id; -export const webhooksApiServiceId = webhooksApi.service.id; +export const graphqlApiServiceId = graphql.service.id; +export const usageApiServiceId = usage.service.id; +export const usageIngestorApiServiceId = usageIngestor.service.id; +export const tokensApiServiceId = tokens.service.id; +export const schemaApiServiceId = schema.service.id; +export const webhooksApiServiceId = webhooks.service.id; export const appId = app.deployment.id; export const publicIp = proxy!.status.loadBalancer.ingress[0].ip; diff --git a/deployment/services/app.ts b/deployment/services/app.ts index e2e2e051e..8363a6b1a 100644 --- a/deployment/services/app.ts +++ b/deployment/services/app.ts @@ -1,61 +1,72 @@ -import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { serviceLocalEndpoint } from '../utils/local-endpoint'; +import { ServiceSecret } from '../utils/secrets'; import { ServiceDeployment } from '../utils/service-deployment'; +import { StripeBilling } from './billing'; import { DbMigrations } from './db-migrations'; +import { Docker } from './docker'; +import { Emails } from './emails'; +import { Environment } from './environment'; +import { GitHubApp } from './github'; import { GraphQL } from './graphql'; - -const appConfig = new pulumi.Config('app'); -const commonConfig = new pulumi.Config('common'); -const githubAppConfig = new pulumi.Config('ghapp'); - -const appEnv = appConfig.requireObject>('env'); -const commonEnv = commonConfig.requireObject>('env'); +import { Sentry } from './sentry'; +import { SlackApp } from './slack-app'; +import { Supertokens } from './supertokens'; +import { Zendesk } from './zendesk'; export type App = ReturnType; +class AppOAuthSecret extends ServiceSecret<{ + clientId: string | pulumi.Output; + clientSecret: string | pulumi.Output; +}> {} + export function deployApp({ - deploymentEnv, graphql, dbMigrations, - release, image, - supertokensConfig, - googleConfig, - githubConfig, - imagePullSecret, - emailsEndpoint, - zendeskSupport, + supertokens, + docker, + emails, + zendesk, + github, + slackApp, + billing, + sentry, + environment, }: { + environment: Environment; image: string; - release: string; - deploymentEnv: DeploymentEnvironment; graphql: GraphQL; dbMigrations: DbMigrations; - imagePullSecret: k8s.core.v1.Secret; - supertokensConfig: { - endpoint: pulumi.Output; - apiKey: pulumi.Output; - }; - googleConfig: { - clientId: pulumi.Output; - clientSecret: pulumi.Output; - }; - githubConfig: { - clientId: pulumi.Output; - clientSecret: pulumi.Output; - }; - emailsEndpoint: pulumi.Output; - zendeskSupport: boolean; + docker: Docker; + supertokens: Supertokens; + emails: Emails; + zendesk: Zendesk; + github: GitHubApp; + slackApp: SlackApp; + billing: StripeBilling; + sentry: Sentry; }) { + const appConfig = new pulumi.Config('app'); + const appEnv = appConfig.requireObject>('env'); + + const oauthConfig = new pulumi.Config('oauth'); + const githubOAuthSecret = new AppOAuthSecret('oauth-github', { + clientId: oauthConfig.requireSecret('githubClient'), + clientSecret: oauthConfig.requireSecret('githubSecret'), + }); + const googleOAuthSecret = new AppOAuthSecret('oauth-google', { + clientId: oauthConfig.requireSecret('googleClient'), + clientSecret: oauthConfig.requireSecret('googleSecret'), + }); + return new ServiceDeployment( 'app', { image, - replicas: isProduction(deploymentEnv) ? 3 : 1, - imagePullSecret, + replicas: environment.isProduction ? 3 : 1, + imagePullSecret: docker.secret, readinessProbe: '/api/health', livenessProbe: '/api/health', startupProbe: { @@ -66,138 +77,38 @@ export function deployApp({ timeoutSeconds: 15, }, availabilityOnEveryNode: true, - env: [ - { name: 'DEPLOYED_DNS', value: deploymentEnv.DEPLOYED_DNS }, - { name: 'NODE_ENV', value: 'production' }, - { - name: 'ENVIRONMENT', - value: deploymentEnv.ENVIRONMENT, - }, - { - name: 'RELEASE', - value: release, - }, - { name: 'SENTRY_DSN', value: commonEnv.SENTRY_DSN }, - { name: 'SENTRY', value: commonEnv.SENTRY_ENABLED }, - { - name: 'GRAPHQL_ENDPOINT', - value: serviceLocalEndpoint(graphql.service).apply(s => `${s}/graphql`), - }, - { - name: 'SERVER_ENDPOINT', - value: serviceLocalEndpoint(graphql.service), - }, - { - name: 'APP_BASE_URL', - value: `https://${deploymentEnv.DEPLOYED_DNS}/`, - }, - { - name: 'INTEGRATION_SLACK', - value: '1', - }, - { - name: 'INTEGRATION_SLACK_CLIENT_ID', - value: appEnv.SLACK_CLIENT_ID, - }, - { - name: 'INTEGRATION_SLACK_CLIENT_SECRET', - value: appEnv.SLACK_CLIENT_SECRET, - }, - { - name: 'INTEGRATION_GITHUB_APP_NAME', - value: githubAppConfig.require('name'), - }, - - { - name: 'STRIPE_PUBLIC_KEY', - value: appEnv.STRIPE_PUBLIC_KEY, - }, - - { - name: 'GA_TRACKING_ID', - value: appEnv.GA_TRACKING_ID, - }, - - { - name: 'CRISP_WEBSITE_ID', - value: appEnv.CRISP_WEBSITE_ID, - }, - - { - name: 'DOCS_URL', - value: 'https://the-guild.dev/graphql/hive/docs', - }, - - { - name: 'GRAPHQL_PERSISTED_OPERATIONS', - value: '1', - }, - - { - name: 'ZENDESK_SUPPORT', - value: zendeskSupport ? '1' : '0', - }, - - // - // AUTH - // - { - name: 'SUPERTOKENS_CONNECTION_URI', - value: supertokensConfig.endpoint, - }, - { - name: 'SUPERTOKENS_API_KEY', - value: supertokensConfig.apiKey, - }, - { - name: 'EMAILS_ENDPOINT', - value: emailsEndpoint, - }, - - // GitHub - { - name: 'AUTH_GITHUB', - value: '1', - }, - { - name: 'AUTH_GITHUB_CLIENT_ID', - value: githubConfig.clientId, - }, - { - name: 'AUTH_GITHUB_CLIENT_SECRET', - value: githubConfig.clientSecret, - }, - // Google - { - name: 'AUTH_GOOGLE', - value: '1', - }, - { - name: 'AUTH_GOOGLE_CLIENT_ID', - value: googleConfig.clientId, - }, - { - name: 'AUTH_GOOGLE_CLIENT_SECRET', - value: googleConfig.clientSecret, - }, - { - name: 'AUTH_REQUIRE_EMAIL_VERIFICATION', - value: '1', - }, - { - name: 'AUTH_ORGANIZATION_OIDC', - value: '1', - }, - // - // Migrations - // - { - name: 'MEMBER_ROLES_DEADLINE', - value: appEnv.MEMBER_ROLES_DEADLINE, - }, - ], + env: { + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', + GRAPHQL_ENDPOINT: serviceLocalEndpoint(graphql.service).apply(s => `${s}/graphql`), + SERVER_ENDPOINT: serviceLocalEndpoint(graphql.service), + APP_BASE_URL: `https://${environment.appDns}/`, + INTEGRATION_SLACK: '1', + INTEGRATION_GITHUB_APP_NAME: github.name, + GA_TRACKING_ID: appEnv.GA_TRACKING_ID, + DOCS_URL: 'https://the-guild.dev/graphql/hive/docs', + GRAPHQL_PERSISTED_OPERATIONS: '1', + ZENDESK_SUPPORT: zendesk.enabled ? '1' : '0', + SUPERTOKENS_CONNECTION_URI: supertokens.localEndpoint, + EMAILS_ENDPOINT: serviceLocalEndpoint(emails.service), + AUTH_GITHUB: '1', + AUTH_GOOGLE: '1', + AUTH_REQUIRE_EMAIL_VERIFICATION: '1', + AUTH_ORGANIZATION_OIDC: '1', + MEMBER_ROLES_DEADLINE: appEnv.MEMBER_ROLES_DEADLINE, + }, port: 3000, }, [graphql.service, graphql.deployment, dbMigrations], - ).deploy(); + ) + .withSecret('INTEGRATION_SLACK_CLIENT_ID', slackApp.secret, 'clientId') + .withSecret('INTEGRATION_SLACK_CLIENT_SECRET', slackApp.secret, 'clientSecret') + .withSecret('STRIPE_PUBLIC_KEY', billing.secret, 'stripePublicKey') + .withSecret('SUPERTOKENS_API_KEY', supertokens.secret, 'apiKey') + .withSecret('AUTH_GITHUB_CLIENT_ID', githubOAuthSecret, 'clientId') + .withSecret('AUTH_GITHUB_CLIENT_SECRET', githubOAuthSecret, 'clientSecret') + .withSecret('AUTH_GOOGLE_CLIENT_ID', googleOAuthSecret, 'clientId') + .withSecret('AUTH_GOOGLE_CLIENT_SECRET', googleOAuthSecret, 'clientSecret') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/billing.ts b/deployment/services/billing.ts index aeba1c73e..59ddf91f9 100644 --- a/deployment/services/billing.ts +++ b/deployment/services/billing.ts @@ -1,66 +1,77 @@ -import { parse } from 'pg-connection-string'; -import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { serviceLocalEndpoint } from '../utils/local-endpoint'; +import { ServiceSecret } from '../utils/secrets'; import { ServiceDeployment } from '../utils/service-deployment'; import { DbMigrations } from './db-migrations'; +import { Docker } from './docker'; +import { Environment } from './environment'; +import { Postgres } from './postgres'; +import { Sentry } from './sentry'; import { UsageEstimator } from './usage-estimation'; -const billingConfig = new pulumi.Config('billing'); -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); -const apiConfig = new pulumi.Config('api'); - export type StripeBillingService = ReturnType; +class StripeSecret extends ServiceSecret<{ + stripePrivateKey: pulumi.Output | string; + stripePublicKey: string | pulumi.Output; +}> {} + export function deployStripeBilling({ - deploymentEnv, + environment, dbMigrations, usageEstimator, image, - release, - imagePullSecret, + docker, + postgres, + sentry, }: { usageEstimator: UsageEstimator; image: string; - release: string; - deploymentEnv: DeploymentEnvironment; + environment: Environment; dbMigrations: DbMigrations; - imagePullSecret: k8s.core.v1.Secret; + docker: Docker; + postgres: Postgres; + sentry: Sentry; }) { - const rawConnectionString = apiConfig.requireSecret('postgresConnectionString'); - const connectionString = rawConnectionString.apply(rawConnectionString => - parse(rawConnectionString), - ); - - return new ServiceDeployment( + const billingConfig = new pulumi.Config('billing'); + const stripeSecret = new StripeSecret('stripe', { + stripePrivateKey: billingConfig.requireSecret('stripePrivateKey'), + stripePublicKey: billingConfig.require('stripePublicKey'), + }); + const { deployment, service } = new ServiceDeployment( 'stripe-billing', { image, - imagePullSecret, - replicas: isProduction(deploymentEnv) ? 3 : 1, + imagePullSecret: docker.secret, + replicas: environment.isProduction ? 3 : 1, readinessProbe: '/_readiness', livenessProbe: '/_health', startupProbe: '/_health', env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - RELEASE: release, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', USAGE_ESTIMATOR_ENDPOINT: serviceLocalEndpoint(usageEstimator.service), - STRIPE_SECRET_KEY: billingConfig.requireSecret('stripePrivateKey'), - POSTGRES_HOST: connectionString.apply(connection => connection.host ?? ''), - POSTGRES_PORT: connectionString.apply(connection => connection.port || '5432'), - POSTGRES_PASSWORD: connectionString.apply(connection => connection.password ?? ''), - POSTGRES_USER: connectionString.apply(connection => connection.user ?? ''), - POSTGRES_DB: connectionString.apply(connection => connection.database ?? ''), - POSTGRES_SSL: connectionString.apply(connection => (connection.ssl ? '1' : '0')), }, exposesMetrics: true, port: 4000, }, [dbMigrations, usageEstimator.service, usageEstimator.deployment], - ).deploy(); + ) + .withSecret('STRIPE_SECRET_KEY', stripeSecret, 'stripePrivateKey') + .withSecret('POSTGRES_HOST', postgres.secret, 'host') + .withSecret('POSTGRES_PORT', postgres.secret, 'port') + .withSecret('POSTGRES_USER', postgres.secret, 'user') + .withSecret('POSTGRES_PASSWORD', postgres.secret, 'password') + .withSecret('POSTGRES_DB', postgres.secret, 'database') + .withSecret('POSTGRES_SSL', postgres.secret, 'ssl') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); + + return { + deployment, + service, + secret: stripeSecret, + }; } + +export type StripeBilling = ReturnType; diff --git a/deployment/services/cf-broker.ts b/deployment/services/cf-broker.ts index 0cda7c39b..ac90cc55b 100644 --- a/deployment/services/cf-broker.ts +++ b/deployment/services/cf-broker.ts @@ -1,35 +1,40 @@ import * as pulumi from '@pulumi/pulumi'; import { CloudflareBroker } from '../utils/cloudflare'; -import { isProduction } from '../utils/helpers'; +import { ServiceSecret } from '../utils/secrets'; +import { Environment } from './environment'; +import { Sentry } from './sentry'; -const commonConfig = new pulumi.Config('common'); -const cfConfig = new pulumi.Config('cloudflareCustom'); -const observabilityConfig = new pulumi.Config('observability'); - -const commonEnv = commonConfig.requireObject>('env'); +export class CloudFlareBrokerSecret extends ServiceSecret<{ + secretSignature: string | pulumi.Output; + baseUrl: string | pulumi.Output; +}> {} export type Broker = ReturnType; export function deployCFBroker({ - rootDns, - envName, - release, + environment, + sentry, }: { - rootDns: string; - envName: string; - release: string; + environment: Environment; + sentry: Sentry; }) { + const commonConfig = new pulumi.Config('common'); + const cfConfig = new pulumi.Config('cloudflareCustom'); + const observabilityConfig = new pulumi.Config('observability'); + const cfBrokerSignature = commonConfig.requireSecret('cfBrokerSignature'); const broker = new CloudflareBroker({ - envName, + envName: environment.envName, zoneId: cfConfig.require('zoneId'), // We can't cdn for staging env, since CF certificate only covers // one level of subdomains. See: https://community.cloudflare.com/t/ssl-handshake-error-cloudflare-proxy/175088 // So for staging env, we are going to use `broker-staging` instead of `broker.staging`. - cdnDnsRecord: isProduction(envName) ? `broker.${rootDns}` : `broker-${rootDns}`, + cdnDnsRecord: environment.isProduction + ? `broker.${environment.rootDns}` + : `broker-${environment.rootDns}`, secretSignature: cfBrokerSignature, - sentryDsn: commonEnv.SENTRY_DSN, - release, + sentryDsn: sentry.enabled && sentry.secret ? sentry.secret.raw.dsn : '', + release: environment.release, loki: observabilityConfig.getBoolean('enabled') ? { endpoint: observabilityConfig.require('lokiEndpoint'), @@ -38,5 +43,16 @@ export function deployCFBroker({ } : null, }); - return broker.deploy(); + + const deployedBroker = broker.deploy(); + + const secret = new CloudFlareBrokerSecret('cloudflare-broker', { + secretSignature: cfBrokerSignature, + baseUrl: deployedBroker.workerBaseUrl, + }); + + return { + broker: deployedBroker, + secret, + }; } diff --git a/deployment/services/cf-cdn.ts b/deployment/services/cf-cdn.ts index 6f9f5b730..28c5d0dbf 100644 --- a/deployment/services/cf-cdn.ts +++ b/deployment/services/cf-cdn.ts @@ -1,41 +1,51 @@ import * as pulumi from '@pulumi/pulumi'; import { CloudflareCDN } from '../utils/cloudflare'; -import { isProduction } from '../utils/helpers'; - -const commonConfig = new pulumi.Config('common'); -const cfConfig = new pulumi.Config('cloudflareCustom'); - -const commonEnv = commonConfig.requireObject>('env'); +import { ServiceSecret } from '../utils/secrets'; +import { Environment } from './environment'; +import { S3 } from './s3'; +import { Sentry } from './sentry'; export type CDN = ReturnType; +export class CDNSecret extends ServiceSecret<{ + authPrivateKey: string | pulumi.Output; + baseUrl: string | pulumi.Output; +}> {} + export function deployCFCDN({ - rootDns, - release, - envName, - s3Config, + environment, + s3, + sentry, }: { - rootDns: string; - envName: string; - release: string; - s3Config: { - endpoint: string; - bucketName: string; - accessKeyId: pulumi.Output; - secretAccessKey: pulumi.Output; - }; + environment: Environment; + s3: S3; + sentry: Sentry; }) { + const cfConfig = new pulumi.Config('cloudflareCustom'); + const cdn = new CloudflareCDN({ - envName, + envName: environment.envName, zoneId: cfConfig.require('zoneId'), // We can't cdn for staging env, since CF certificate only covers // one level of subdomains. See: https://community.cloudflare.com/t/ssl-handshake-error-cloudflare-proxy/175088 // So for staging env, we are going to use `cdn-staging` instead of `cdn.staging`. - cdnDnsRecord: isProduction(envName) ? `cdn.${rootDns}` : `cdn-${rootDns}`, - sentryDsn: commonEnv.SENTRY_DSN, - release, - s3Config, + cdnDnsRecord: environment.isProduction + ? `cdn.${environment.rootDns}` + : `cdn-${environment.rootDns}`, + sentryDsn: sentry.enabled && sentry.secret ? sentry.secret?.raw.dsn : '', + release: environment.release, + s3, }); - return cdn.deploy(); + const deployedCdn = cdn.deploy(); + const cdnConfig = new pulumi.Config('cdn'); + const secret = new CDNSecret('cdn', { + authPrivateKey: cdnConfig.requireSecret('authPrivateKey'), + baseUrl: deployedCdn.workerBaseUrl, + }); + + return { + cdn: deployedCdn, + secret, + }; } diff --git a/deployment/services/clickhouse.ts b/deployment/services/clickhouse.ts index b70273e33..bd74f0c0c 100644 --- a/deployment/services/clickhouse.ts +++ b/deployment/services/clickhouse.ts @@ -1,30 +1,28 @@ import * as pulumi from '@pulumi/pulumi'; - -const clickhouseConfig = new pulumi.Config('clickhouse'); +import { ServiceSecret } from '../utils/secrets'; export type Clickhouse = ReturnType; -type ClickhouseConfig = { - protocol: pulumi.Output | string; - host: pulumi.Output | string; - port: pulumi.Output | string; - username: pulumi.Output | string; - password: pulumi.Output; -}; +export class ClickhouseConnectionSecret extends ServiceSecret<{ + host: string | pulumi.Output; + port: string | pulumi.Output; + username: string | pulumi.Output; + password: string | pulumi.Output; + protocol: string | pulumi.Output; +}> {} -function getRemoteClickhouseConfig(): ClickhouseConfig { - return { +export function deployClickhouse() { + const clickhouseConfig = new pulumi.Config('clickhouse'); + const secret = new ClickhouseConnectionSecret('clickhouse', { host: clickhouseConfig.require('host'), port: clickhouseConfig.require('port'), username: clickhouseConfig.require('username'), password: clickhouseConfig.requireSecret('password'), protocol: clickhouseConfig.require('protocol'), - }; -} + }); -export function deployClickhouse() { return { - config: getRemoteClickhouseConfig(), + secret, deployment: null, service: null, }; diff --git a/deployment/services/database-cleanup.ts b/deployment/services/database-cleanup.ts index cb7b3e81f..4e7ee165c 100644 --- a/deployment/services/database-cleanup.ts +++ b/deployment/services/database-cleanup.ts @@ -1,13 +1,12 @@ import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { ServiceDeployment } from '../utils/service-deployment'; +import { Environment } from './environment'; const apiConfig = new pulumi.Config('api'); -export function deployDatabaseCleanupJob(options: { deploymentEnv: DeploymentEnvironment }) { - if (isProduction(options.deploymentEnv)) { +export function deployDatabaseCleanupJob(options: { environment: Environment }) { + if (options.environment.isProduction) { throw new Error('Database cleanup job is not allowed in "production" environment!'); } diff --git a/deployment/services/db-migrations.ts b/deployment/services/db-migrations.ts index 4094ec49e..9614406ff 100644 --- a/deployment/services/db-migrations.ts +++ b/deployment/services/db-migrations.ts @@ -1,75 +1,47 @@ -import { parse } from 'pg-connection-string'; -import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; import { ServiceDeployment } from '../utils/service-deployment'; +import { CDN } from './cf-cdn'; import { Clickhouse } from './clickhouse'; -import { Kafka } from './kafka'; - -const apiConfig = new pulumi.Config('api'); +import { Docker } from './docker'; +import { Environment } from './environment'; +import { Postgres } from './postgres'; +import { S3 } from './s3'; export type DbMigrations = ReturnType; export function deployDbMigrations({ - deploymentEnv, + environment, clickhouse, - kafka, + s3, image, - imagePullSecret, dependencies, force, - s3, - cdnAuthPrivateKey, + docker, + postgres, + cdn, }: { - deploymentEnv: DeploymentEnvironment; + docker: Docker; + postgres: Postgres; clickhouse: Clickhouse; - kafka: Kafka; + s3: S3; + cdn: CDN; + environment: Environment; image: string; - imagePullSecret: k8s.core.v1.Secret; dependencies?: pulumi.Resource[]; force?: boolean; - s3: { - accessKeyId: string | pulumi.Output; - secretAccessKey: string | pulumi.Output; - endpoint: string | pulumi.Output; - bucketName: string | pulumi.Output; - }; - cdnAuthPrivateKey: pulumi.Output; }) { - const rawConnectionString = apiConfig.requireSecret('postgresConnectionString'); - const connectionString = rawConnectionString.apply(rawConnectionString => - parse(rawConnectionString), - ); - const { job } = new ServiceDeployment( 'db-migrations', { - imagePullSecret, + imagePullSecret: docker.secret, image, env: { - POSTGRES_HOST: connectionString.apply(connection => connection.host ?? ''), - POSTGRES_PORT: connectionString.apply(connection => connection.port || '5432'), - POSTGRES_PASSWORD: connectionString.apply(connection => connection.password ?? ''), - POSTGRES_USER: connectionString.apply(connection => connection.user ?? ''), - POSTGRES_DB: connectionString.apply(connection => connection.database ?? ''), - POSTGRES_SSL: connectionString.apply(connection => (connection.ssl ? '1' : '0')), + ...environment.envVars, MIGRATOR: 'up', CLICKHOUSE_MIGRATOR: 'up', - CLICKHOUSE_HOST: clickhouse.config.host, - CLICKHOUSE_PORT: clickhouse.config.port, - CLICKHOUSE_USERNAME: clickhouse.config.username, - CLICKHOUSE_PASSWORD: clickhouse.config.password, - CLICKHOUSE_PROTOCOL: clickhouse.config.protocol, CLICKHOUSE_MIGRATOR_GRAPHQL_HIVE_CLOUD: '1', - KAFKA_BROKER: kafka.config.endpoint, TS_NODE_TRANSPILE_ONLY: 'true', RUN_S3_LEGACY_CDN_KEY_IMPORT: '1', - S3_ACCESS_KEY_ID: s3.accessKeyId, - S3_SECRET_ACCESS_KEY: s3.secretAccessKey, - S3_ENDPOINT: s3.endpoint, - S3_BUCKET_NAME: s3.bucketName, - CDN_AUTH_PRIVATE_KEY: cdnAuthPrivateKey, - ...deploymentEnv, // Change to this env var will lead to force rerun of the migration job // Since K8s job are immutable, we can't edit or ask K8s to re-run a Job, so we are doing a // pseudo change to an env var, which causes Pulumi to re-create the Job. @@ -78,7 +50,24 @@ export function deployDbMigrations({ }, [clickhouse.deployment, clickhouse.service, ...(dependencies || [])], clickhouse.service, - ).deployAsJob(); + ) + .withSecret('POSTGRES_HOST', postgres.secret, 'host') + .withSecret('POSTGRES_PORT', postgres.secret, 'port') + .withSecret('POSTGRES_USER', postgres.secret, 'user') + .withSecret('POSTGRES_PASSWORD', postgres.secret, 'password') + .withSecret('POSTGRES_DB', postgres.secret, 'database') + .withSecret('POSTGRES_SSL', postgres.secret, 'ssl') + .withSecret('CLICKHOUSE_HOST', clickhouse.secret, 'host') + .withSecret('CLICKHOUSE_PORT', clickhouse.secret, 'port') + .withSecret('CLICKHOUSE_USERNAME', clickhouse.secret, 'username') + .withSecret('CLICKHOUSE_PASSWORD', clickhouse.secret, 'password') + .withSecret('CLICKHOUSE_PROTOCOL', clickhouse.secret, 'protocol') + .withSecret('S3_ACCESS_KEY_ID', s3.secret, 'accessKeyId') + .withSecret('S3_SECRET_ACCESS_KEY', s3.secret, 'secretAccessKey') + .withSecret('S3_BUCKET_NAME', s3.secret, 'bucket') + .withSecret('S3_ENDPOINT', s3.secret, 'endpoint') + .withSecret('CDN_AUTH_PRIVATE_KEY', cdn.secret, 'authPrivateKey') + .deployAsJob(); return job; } diff --git a/deployment/services/docker.ts b/deployment/services/docker.ts new file mode 100644 index 000000000..27c3729b9 --- /dev/null +++ b/deployment/services/docker.ts @@ -0,0 +1,21 @@ +import * as pulumi from '@pulumi/pulumi'; +import { createDockerImageFactory } from '../utils/docker-images'; + +export function configureDocker() { + const dockerConfig = new pulumi.Config('docker'); + const dockerImages = createDockerImageFactory({ + registryHostname: dockerConfig.require('registryUrl'), + imagesPrefix: dockerConfig.require('imagesPrefix'), + }); + + const imagePullSecret = dockerImages.createRepositorySecret( + dockerConfig.requireSecret('registryAuthBase64'), + ); + + return { + secret: imagePullSecret, + factory: dockerImages, + }; +} + +export type Docker = ReturnType; diff --git a/deployment/services/emails.ts b/deployment/services/emails.ts index beadf0f02..361310a92 100644 --- a/deployment/services/emails.ts +++ b/deployment/services/emails.ts @@ -1,53 +1,50 @@ -import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { serviceLocalEndpoint } from '../utils/local-endpoint'; +import { ServiceSecret } from '../utils/secrets'; import { ServiceDeployment } from '../utils/service-deployment'; +import { Docker } from './docker'; +import { Environment } from './environment'; import { Redis } from './redis'; - -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); +import { Sentry } from './sentry'; export type Emails = ReturnType; +class PostmarkSecret extends ServiceSecret<{ + token: pulumi.Output | string; + from: string; + messageStream: string; +}> {} + export function deployEmails({ - deploymentEnv, + environment, redis, heartbeat, - email, - release, image, - imagePullSecret, + docker, + sentry, }: { - release: string; + environment: Environment; image: string; - deploymentEnv: DeploymentEnvironment; redis: Redis; + docker: Docker; heartbeat?: string; - email: { - token: pulumi.Output; - from: string; - messageStream: string; - }; - imagePullSecret: k8s.core.v1.Secret; + sentry: Sentry; }) { + const emailConfig = new pulumi.Config('email'); + const postmarkSecret = new PostmarkSecret('postmark', { + token: emailConfig.requireSecret('token'), + from: emailConfig.require('from'), + messageStream: emailConfig.require('messageStream'), + }); + const { deployment, service } = new ServiceDeployment( 'emails-service', { - imagePullSecret, + imagePullSecret: docker.secret, env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - RELEASE: release, - REDIS_HOST: redis.config.host, - REDIS_PORT: String(redis.config.port), - REDIS_PASSWORD: redis.config.password, - EMAIL_FROM: email.from, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', EMAIL_PROVIDER: 'postmark', - EMAIL_PROVIDER_POSTMARK_TOKEN: email.token, - EMAIL_PROVIDER_POSTMARK_MESSAGE_STREAM: email.messageStream, HEARTBEAT_ENDPOINT: heartbeat ?? '', }, readinessProbe: '/_readiness', @@ -55,10 +52,18 @@ export function deployEmails({ startupProbe: '/_health', exposesMetrics: true, image, - replicas: isProduction(deploymentEnv) ? 3 : 1, + replicas: environment.isProduction ? 3 : 1, }, [redis.deployment, redis.service], - ).deploy(); + ) + .withSecret('REDIS_HOST', redis.secret, 'host') + .withSecret('REDIS_PORT', redis.secret, 'port') + .withSecret('REDIS_PASSWORD', redis.secret, 'password') + .withSecret('EMAIL_FROM', postmarkSecret, 'from') + .withSecret('EMAIL_PROVIDER_POSTMARK_TOKEN', postmarkSecret, 'token') + .withSecret('EMAIL_PROVIDER_POSTMARK_MESSAGE_STREAM', postmarkSecret, 'messageStream') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); return { deployment, service, localEndpoint: serviceLocalEndpoint(service) }; } diff --git a/deployment/services/environment.ts b/deployment/services/environment.ts new file mode 100644 index 000000000..3c7cd4146 --- /dev/null +++ b/deployment/services/environment.ts @@ -0,0 +1,45 @@ +import { Config, Output } from '@pulumi/pulumi'; +import { ServiceSecret } from '../utils/secrets'; + +export class DataEncryptionSecret extends ServiceSecret<{ + encryptionPrivateKey: string | Output; +}> {} + +export function prepareEnvironment(input: { + release: string; + rootDns: string; + environment: string; +}) { + const commonConfig = new Config('common'); + + const encryptionSecret = new DataEncryptionSecret('data-encryption', { + encryptionPrivateKey: commonConfig.requireSecret('encryptionSecret'), + }); + + const env = + input.environment === 'production' || input.environment === 'prod' + ? 'production' + : input.environment; + + const appDns = `app.${input.rootDns}`; + + return { + envVars: { + NODE_ENV: 'production', + LOG_LEVEL: 'debug', + DEPLOYED_DNS: appDns, + ENVIRONMENT: input.environment, + RELEASE: input.release, + }, + envName: env, + isProduction: env === 'production', + isStaging: env === 'staging', + isDev: env === 'dev', + encryptionSecret, + release: input.release, + appDns, + rootDns: input.rootDns, + }; +} + +export type Environment = ReturnType; diff --git a/deployment/services/github.ts b/deployment/services/github.ts new file mode 100644 index 000000000..5bfdf329e --- /dev/null +++ b/deployment/services/github.ts @@ -0,0 +1,22 @@ +import { Config, Output } from '@pulumi/pulumi'; +import { ServiceSecret } from '../utils/secrets'; + +class GitHubIntegrationSecret extends ServiceSecret<{ + appId: string | Output; + privateKey: string | Output; +}> {} + +export function configureGithubApp() { + const githubAppConfig = new Config('ghapp'); + const githubSecret = new GitHubIntegrationSecret('gitub-app', { + appId: githubAppConfig.require('id'), + privateKey: githubAppConfig.requireSecret('key'), + }); + + return { + name: githubAppConfig.require('name'), + secret: githubSecret, + }; +} + +export type GitHubApp = ReturnType; diff --git a/deployment/services/graphql.ts b/deployment/services/graphql.ts index fff385cd4..4a8c49167 100644 --- a/deployment/services/graphql.ts +++ b/deployment/services/graphql.ts @@ -1,40 +1,34 @@ -import { parse } from 'pg-connection-string'; -import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { Output } from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { serviceLocalEndpoint } from '../utils/local-endpoint'; import { ServiceDeployment } from '../utils/service-deployment'; import { StripeBillingService } from './billing'; import { CDN } from './cf-cdn'; import { Clickhouse } from './clickhouse'; import { DbMigrations } from './db-migrations'; +import { Docker } from './docker'; import { Emails } from './emails'; +import { Environment } from './environment'; +import { GitHubApp } from './github'; import { SchemaPolicy } from './policy'; +import { Postgres } from './postgres'; import { RateLimitService } from './rate-limit'; import { Redis } from './redis'; +import { S3 } from './s3'; import { Schema } from './schema'; +import { Sentry } from './sentry'; +import { Supertokens } from './supertokens'; import { Tokens } from './tokens'; import { Usage } from './usage'; import { UsageEstimator } from './usage-estimation'; import { Webhooks } from './webhooks'; - -const commonConfig = new pulumi.Config('common'); -const cloudflareConfig = new pulumi.Config('cloudflare'); -const apiConfig = new pulumi.Config('api'); -const githubAppConfig = new pulumi.Config('ghapp'); - -const commonEnv = commonConfig.requireObject>('env'); -const apiEnv = apiConfig.requireObject>('env'); +import { Zendesk } from './zendesk'; export type GraphQL = ReturnType; export function deployGraphQL({ clickhouse, - release, image, - deploymentEnv, + environment, tokens, webhooks, schema, @@ -47,152 +41,135 @@ export function deployGraphQL({ rateLimit, billing, emails, - supertokensConfig, - s3Config, - zendeskConfig, - imagePullSecret, - cdnAuthPrivateKey, + supertokens, + s3, + zendesk, + docker, + postgres, + githubApp, + sentry, }: { - release: string; + githubApp: GitHubApp; + postgres: Postgres; image: string; clickhouse: Clickhouse; - deploymentEnv: DeploymentEnvironment; + environment: Environment; tokens: Tokens; webhooks: Webhooks; schema: Schema; schemaPolicy: SchemaPolicy; redis: Redis; cdn: CDN; - cdnAuthPrivateKey: Output; + s3: S3; usage: Usage; usageEstimator: UsageEstimator; dbMigrations: DbMigrations; rateLimit: RateLimitService; billing: StripeBillingService; emails: Emails; - supertokensConfig: { - endpoint: Output; - apiKey: Output; - }; - s3Config: { - endpoint: string; - bucketName: string; - accessKeyId: Output; - secretAccessKey: Output; - }; - zendeskConfig: { - username: string; - password: Output; - subdomain: string; - } | null; - imagePullSecret: k8s.core.v1.Secret; + supertokens: Supertokens; + zendesk: Zendesk; + docker: Docker; + sentry: Sentry; }) { - const rawConnectionString = apiConfig.requireSecret('postgresConnectionString'); - const connectionString = rawConnectionString.apply(rawConnectionString => - parse(rawConnectionString), - ); + const apiConfig = new pulumi.Config('api'); + const apiEnv = apiConfig.requireObject>('env'); - return new ServiceDeployment( - 'graphql-api', - { - imagePullSecret, - image, - replicas: isProduction(deploymentEnv) ? 3 : 1, - pdb: true, - readinessProbe: '/_readiness', - livenessProbe: '/_health', - startupProbe: { - endpoint: '/_health', - initialDelaySeconds: 60, - failureThreshold: 10, - periodSeconds: 15, - timeoutSeconds: 15, + return ( + new ServiceDeployment( + 'graphql-api', + { + imagePullSecret: docker.secret, + image, + replicas: environment.isProduction ? 3 : 1, + pdb: true, + readinessProbe: '/_readiness', + livenessProbe: '/_health', + startupProbe: { + endpoint: '/_health', + initialDelaySeconds: 60, + failureThreshold: 10, + periodSeconds: 15, + timeoutSeconds: 15, + }, + availabilityOnEveryNode: true, + env: { + ...environment.envVars, + ...apiEnv, + SENTRY: sentry.enabled ? '1' : '0', + REQUEST_LOGGING: '0', // disabled + BILLING_ENDPOINT: serviceLocalEndpoint(billing.service), + TOKENS_ENDPOINT: serviceLocalEndpoint(tokens.service), + WEBHOOKS_ENDPOINT: serviceLocalEndpoint(webhooks.service), + SCHEMA_ENDPOINT: serviceLocalEndpoint(schema.service), + SCHEMA_POLICY_ENDPOINT: serviceLocalEndpoint(schemaPolicy.service), + HIVE_USAGE_ENDPOINT: serviceLocalEndpoint(usage.service), + RATE_LIMIT_ENDPOINT: serviceLocalEndpoint(rateLimit.service), + EMAILS_ENDPOINT: serviceLocalEndpoint(emails.service), + USAGE_ESTIMATOR_ENDPOINT: serviceLocalEndpoint(usageEstimator.service), + WEB_APP_URL: `https://${environment.appDns}`, + CDN_CF: '1', + HIVE: '1', + HIVE_REPORTING: '1', + HIVE_USAGE: '1', + HIVE_REPORTING_ENDPOINT: 'http://0.0.0.0:4000/graphql', + ZENDESK_SUPPORT: zendesk.enabled ? '1' : '0', + INTEGRATION_GITHUB: '1', + SUPERTOKENS_CONNECTION_URI: supertokens.localEndpoint, + AUTH_ORGANIZATION_OIDC: '1', + GRAPHQL_PERSISTED_OPERATIONS_PATH: './persisted-operations.json', + }, + exposesMetrics: true, + port: 4000, }, - availabilityOnEveryNode: true, - env: { - ...apiEnv, - ...deploymentEnv, - ...apiConfig.requireObject>('env'), - ...commonEnv, - RELEASE: release, - SENTRY: commonEnv.SENTRY_ENABLED, - // Logging - REQUEST_LOGGING: '0', // disabled - // ClickHouse - CLICKHOUSE_PROTOCOL: clickhouse.config.protocol, - CLICKHOUSE_HOST: clickhouse.config.host, - CLICKHOUSE_PORT: clickhouse.config.port, - CLICKHOUSE_USERNAME: clickhouse.config.username, - CLICKHOUSE_PASSWORD: clickhouse.config.password, - // Redis - REDIS_HOST: redis.config.host, - REDIS_PORT: String(redis.config.port), - REDIS_PASSWORD: redis.config.password, - // PG - POSTGRES_HOST: connectionString.apply(connection => connection.host ?? ''), - POSTGRES_PORT: connectionString.apply(connection => connection.port || '5432'), - POSTGRES_PASSWORD: connectionString.apply(connection => connection.password ?? ''), - POSTGRES_USER: connectionString.apply(connection => connection.user ?? ''), - POSTGRES_DB: connectionString.apply(connection => connection.database ?? ''), - POSTGRES_SSL: connectionString.apply(connection => (connection.ssl ? '1' : '0')), - // S3 - S3_ENDPOINT: s3Config.endpoint, - S3_ACCESS_KEY_ID: s3Config.accessKeyId, - S3_SECRET_ACCESS_KEY: s3Config.secretAccessKey, - S3_BUCKET_NAME: s3Config.bucketName, - BILLING_ENDPOINT: serviceLocalEndpoint(billing.service), - TOKENS_ENDPOINT: serviceLocalEndpoint(tokens.service), - WEBHOOKS_ENDPOINT: serviceLocalEndpoint(webhooks.service), - SCHEMA_ENDPOINT: serviceLocalEndpoint(schema.service), - SCHEMA_POLICY_ENDPOINT: serviceLocalEndpoint(schemaPolicy.service), - WEB_APP_URL: `https://${deploymentEnv.DEPLOYED_DNS}`, - // CDN - CDN_CF: '1', - CDN_CF_BASE_URL: cdn.workerBaseUrl, - CDN_AUTH_PRIVATE_KEY: cdnAuthPrivateKey, - // Hive - HIVE: '1', - HIVE_REPORTING: '1', - HIVE_USAGE: '1', - HIVE_USAGE_ENDPOINT: serviceLocalEndpoint(usage.service), - HIVE_REPORTING_ENDPOINT: 'http://0.0.0.0:4000/graphql', - // Zendesk - ...(zendeskConfig - ? { - ZENDESK_SUPPORT: '1', - ZENDESK_USERNAME: zendeskConfig.username, - ZENDESK_PASSWORD: zendeskConfig.password, - ZENDESK_SUBDOMAIN: zendeskConfig.subdomain, - } - : { - ZENDESK_SUPPORT: '0', - }), - // - USAGE_ESTIMATOR_ENDPOINT: serviceLocalEndpoint(usageEstimator.service), - INTEGRATION_GITHUB: '1', - INTEGRATION_GITHUB_APP_ID: githubAppConfig.require('id'), - INTEGRATION_GITHUB_APP_PRIVATE_KEY: githubAppConfig.requireSecret('key'), - RATE_LIMIT_ENDPOINT: serviceLocalEndpoint(rateLimit.service), - EMAILS_ENDPOINT: serviceLocalEndpoint(emails.service), - ENCRYPTION_SECRET: commonConfig.requireSecret('encryptionSecret'), - // Auth - SUPERTOKENS_CONNECTION_URI: supertokensConfig.endpoint, - SUPERTOKENS_API_KEY: supertokensConfig.apiKey, - AUTH_ORGANIZATION_OIDC: '1', - // Various - GRAPHQL_PERSISTED_OPERATIONS_PATH: './persisted-operations.json', - }, - exposesMetrics: true, - port: 4000, - }, - [ - dbMigrations, - redis.deployment, - redis.service, - clickhouse.deployment, - clickhouse.service, - rateLimit.deployment, - rateLimit.service, - ], - ).deploy(); + [ + dbMigrations, + redis.deployment, + redis.service, + clickhouse.deployment, + clickhouse.service, + rateLimit.deployment, + rateLimit.service, + ], + ) + // GitHub App + .withSecret('INTEGRATION_GITHUB_APP_ID', githubApp.secret, 'appId') + .withSecret('INTEGRATION_GITHUB_APP_PRIVATE_KEY', githubApp.secret, 'privateKey') + // Clickhouse + .withSecret('CLICKHOUSE_HOST', clickhouse.secret, 'host') + .withSecret('CLICKHOUSE_PORT', clickhouse.secret, 'port') + .withSecret('CLICKHOUSE_USERNAME', clickhouse.secret, 'username') + .withSecret('CLICKHOUSE_PASSWORD', clickhouse.secret, 'password') + .withSecret('CLICKHOUSE_PROTOCOL', clickhouse.secret, 'protocol') + // Redis + .withSecret('REDIS_HOST', redis.secret, 'host') + .withSecret('REDIS_PORT', redis.secret, 'port') + .withSecret('REDIS_PASSWORD', redis.secret, 'password') + // PG + .withSecret('POSTGRES_HOST', postgres.secret, 'host') + .withSecret('POSTGRES_PORT', postgres.secret, 'port') + .withSecret('POSTGRES_USER', postgres.secret, 'user') + .withSecret('POSTGRES_PASSWORD', postgres.secret, 'password') + .withSecret('POSTGRES_DB', postgres.secret, 'database') + .withSecret('POSTGRES_SSL', postgres.secret, 'ssl') + // CDN + .withSecret('CDN_AUTH_PRIVATE_KEY', cdn.secret, 'authPrivateKey') + .withSecret('CDN_CF_BASE_URL', cdn.secret, 'baseUrl') + // S3 + .withSecret('S3_ACCESS_KEY_ID', s3.secret, 'accessKeyId') + .withSecret('S3_SECRET_ACCESS_KEY', s3.secret, 'secretAccessKey') + .withSecret('S3_BUCKET_NAME', s3.secret, 'bucket') + .withSecret('S3_ENDPOINT', s3.secret, 'endpoint') + // Supertokens + .withSecret('SUPERTOKENS_API_KEY', supertokens.secret, 'apiKey') + // Zendesk + .withConditionalSecret(zendesk.enabled, 'ZENDESK_SUBDOMAIN', zendesk.secret, 'subdomain') + .withConditionalSecret(zendesk.enabled, 'ZENDESK_USERNAME', zendesk.secret, 'username') + .withConditionalSecret(zendesk.enabled, 'ZENDESK_PASSWORD', zendesk.secret, 'password') + // Sentry + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + // Other + .withSecret('ENCRYPTION_SECRET', environment.encryptionSecret, 'encryptionPrivateKey') + .deploy() + ); } diff --git a/deployment/services/kafka.ts b/deployment/services/kafka.ts index c38320f00..f81a6e83f 100644 --- a/deployment/services/kafka.ts +++ b/deployment/services/kafka.ts @@ -1,27 +1,34 @@ import * as pulumi from '@pulumi/pulumi'; +import { ServiceSecret } from '../utils/secrets'; export type Kafka = ReturnType; +export class KafkaSecret extends ServiceSecret<{ + ssl: '0' | '1' | pulumi.Output<'0' | '1'>; + saslUsername: string | pulumi.Output; + saslPassword: string | pulumi.Output; + endpoint: string | pulumi.Output; +}> {} + export function deployKafka() { const eventhubConfig = new pulumi.Config('eventhub'); + const secret = new KafkaSecret('kafka', { + ssl: '1', + saslUsername: '$ConnectionString', + saslPassword: eventhubConfig.requireSecret('key'), + endpoint: eventhubConfig.require('endpoint'), + }); return { - connectionEnv: { - KAFKA_SSL: '1', - KAFKA_SASL_MECHANISM: 'plain', - KAFKA_CONCURRENCY: '1', - KAFKA_SASL_USERNAME: '$ConnectionString', - KAFKA_SASL_PASSWORD: eventhubConfig.requireSecret('key'), - } as Record | string>, + secret, config: { - endpoint: eventhubConfig.require('endpoint'), + saslMechanism: 'plain', + concurrency: '1', bufferSize: eventhubConfig.require('bufferSize'), bufferInterval: eventhubConfig.require('bufferInterval'), bufferDynamic: eventhubConfig.require('bufferDynamic'), topic: eventhubConfig.require('topic'), consumerGroup: eventhubConfig.require('consumerGroup'), }, - service: null, - deployment: null, }; } diff --git a/deployment/services/observability.ts b/deployment/services/observability.ts index c18d610af..27d305348 100644 --- a/deployment/services/observability.ts +++ b/deployment/services/observability.ts @@ -2,11 +2,13 @@ import * as pulumi from '@pulumi/pulumi'; import { Observability } from '../utils/observability'; import { deployGrafana } from './grafana'; -const observabilityConfig = new pulumi.Config('observability'); - export function deployMetrics(config: { envName: string }) { + const observabilityConfig = new pulumi.Config('observability'); + if (!observabilityConfig.getBoolean('enabled')) { - return; + return { + enabled: false, + }; } const observability = new Observability(config.envName, { @@ -25,5 +27,6 @@ export function deployMetrics(config: { envName: string }) { return { observability: observability.deploy(), grafana: deployGrafana(config.envName), + enabled: true, }; } diff --git a/deployment/services/policy.ts b/deployment/services/policy.ts index a33e4293c..af1911857 100644 --- a/deployment/services/policy.ts +++ b/deployment/services/policy.ts @@ -1,39 +1,35 @@ -import * as k8s from '@pulumi/kubernetes'; -import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { ServiceDeployment } from '../utils/service-deployment'; - -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); +import { Docker } from './docker'; +import { Environment } from './environment'; +import { Sentry } from './sentry'; export type SchemaPolicy = ReturnType; export function deploySchemaPolicy({ - deploymentEnv, - release, + environment, image, - imagePullSecret, + docker, + sentry, }: { image: string; - release: string; - deploymentEnv: DeploymentEnvironment; - imagePullSecret: k8s.core.v1.Secret; + environment: Environment; + docker: Docker; + sentry: Sentry; }) { return new ServiceDeployment('schema-policy-service', { image, - imagePullSecret, + imagePullSecret: docker.secret, env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - RELEASE: release, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', }, readinessProbe: '/_readiness', livenessProbe: '/_health', startupProbe: '/_health', exposesMetrics: true, - replicas: isProduction(deploymentEnv) ? 3 : 1, + replicas: environment.isProduction ? 3 : 1, pdb: true, - }).deploy(); + }) + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/postgres.ts b/deployment/services/postgres.ts new file mode 100644 index 000000000..07eb42183 --- /dev/null +++ b/deployment/services/postgres.ts @@ -0,0 +1,39 @@ +import { parse } from 'pg-connection-string'; +import * as pulumi from '@pulumi/pulumi'; +import { ServiceSecret } from '../utils/secrets'; + +export class PostgresConnectionSecret extends ServiceSecret<{ + host: string | pulumi.Output; + port: string | pulumi.Output; + user: string | pulumi.Output; + password: string | pulumi.Output; + database: string | pulumi.Output; + ssl: '0' | '1' | pulumi.Output<'0' | '1'>; + connectionString: pulumi.Output; + connectionStringPostgresql: pulumi.Output; +}> {} + +export function deployPostgres() { + const postgresConfig = new pulumi.Config('postgres'); + const rawConnectionString = postgresConfig.requireSecret('connectionString'); + const connectionString = rawConnectionString.apply(rawConnectionString => + parse(rawConnectionString), + ); + + const secret = new PostgresConnectionSecret('postgres', { + connectionString: rawConnectionString, + connectionStringPostgresql: rawConnectionString.apply(str => + str.replace('postgres://', 'postgresql://'), + ), + host: connectionString.apply(connection => connection.host ?? ''), + port: connectionString.apply(connection => connection.port || '5432'), + user: connectionString.apply(connection => connection.user ?? ''), + password: connectionString.apply(connection => connection.password ?? ''), + database: connectionString.apply(connection => connection.database ?? ''), + ssl: connectionString.apply(connection => (connection.ssl ? '1' : '0')), + }); + + return { secret }; +} + +export type Postgres = ReturnType; diff --git a/deployment/services/proxy.ts b/deployment/services/proxy.ts index b7fb9a5a6..7b099e7f4 100644 --- a/deployment/services/proxy.ts +++ b/deployment/services/proxy.ts @@ -1,34 +1,31 @@ import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; import { CertManager } from '../utils/cert-manager'; -import { isProduction } from '../utils/helpers'; import { Proxy } from '../utils/reverse-proxy'; import { App } from './app'; +import { Environment } from './environment'; import { GraphQL } from './graphql'; import { Usage } from './usage'; -const commonConfig = new pulumi.Config('common'); - export function deployProxy({ - appHostname, graphql, app, usage, - deploymentEnv, + environment, }: { - deploymentEnv: DeploymentEnvironment; - appHostname: string; + environment: Environment; graphql: GraphQL; app: App; usage: Usage; }) { const { tlsIssueName } = new CertManager().deployCertManagerAndIssuer(); + const commonConfig = new pulumi.Config('common'); + return new Proxy(tlsIssueName, { address: commonConfig.get('staticIp'), aksReservedIpResourceGroup: commonConfig.get('aksReservedIpResourceGroup'), }) - .deployProxy({ replicas: isProduction(deploymentEnv) ? 3 : 1 }) - .registerService({ record: appHostname }, [ + .deployProxy({ replicas: environment.isProduction ? 3 : 1 }) + .registerService({ record: environment.appDns }, [ { name: 'app', path: '/', diff --git a/deployment/services/rate-limit.ts b/deployment/services/rate-limit.ts index ba3edebdc..74947a6c0 100644 --- a/deployment/services/rate-limit.ts +++ b/deployment/services/rate-limit.ts @@ -1,71 +1,62 @@ -import { parse } from 'pg-connection-string'; -import * as k8s from '@pulumi/kubernetes'; -import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { serviceLocalEndpoint } from '../utils/local-endpoint'; import { ServiceDeployment } from '../utils/service-deployment'; import { DbMigrations } from './db-migrations'; +import { Docker } from './docker'; import { Emails } from './emails'; +import { Environment } from './environment'; +import { Postgres } from './postgres'; +import { Sentry } from './sentry'; import { UsageEstimator } from './usage-estimation'; -const rateLimitConfig = new pulumi.Config('rateLimit'); -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); -const apiConfig = new pulumi.Config('api'); - export type RateLimitService = ReturnType; export function deployRateLimit({ - deploymentEnv, + environment, dbMigrations, usageEstimator, emails, - release, image, - imagePullSecret, + docker, + postgres, + sentry, }: { usageEstimator: UsageEstimator; - deploymentEnv: DeploymentEnvironment; + environment: Environment; dbMigrations: DbMigrations; emails: Emails; - release: string; image: string; - imagePullSecret: k8s.core.v1.Secret; + docker: Docker; + postgres: Postgres; + sentry: Sentry; }) { - const rawConnectionString = apiConfig.requireSecret('postgresConnectionString'); - const connectionString = rawConnectionString.apply(rawConnectionString => - parse(rawConnectionString), - ); - return new ServiceDeployment( 'rate-limiter', { - imagePullSecret, - replicas: isProduction(deploymentEnv) ? 3 : 1, + imagePullSecret: docker.secret, + replicas: environment.isProduction ? 3 : 1, readinessProbe: '/_readiness', livenessProbe: '/_health', startupProbe: '/_health', env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - LIMIT_CACHE_UPDATE_INTERVAL_MS: rateLimitConfig.require('updateIntervalMs'), - RELEASE: release, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', + LIMIT_CACHE_UPDATE_INTERVAL_MS: environment.isProduction ? '60000' : '86400000', USAGE_ESTIMATOR_ENDPOINT: serviceLocalEndpoint(usageEstimator.service), EMAILS_ENDPOINT: serviceLocalEndpoint(emails.service), - POSTGRES_HOST: connectionString.apply(connection => connection.host ?? ''), - POSTGRES_PORT: connectionString.apply(connection => connection.port || '5432'), - POSTGRES_PASSWORD: connectionString.apply(connection => connection.password ?? ''), - POSTGRES_USER: connectionString.apply(connection => connection.user ?? ''), - POSTGRES_DB: connectionString.apply(connection => connection.database ?? ''), - POSTGRES_SSL: connectionString.apply(connection => (connection.ssl ? '1' : '0')), - WEB_APP_URL: `https://${deploymentEnv.DEPLOYED_DNS}/`, + WEB_APP_URL: `https://${environment.appDns}/`, }, exposesMetrics: true, port: 4000, image, }, [dbMigrations, usageEstimator.service, usageEstimator.deployment], - ).deploy(); + ) + .withSecret('POSTGRES_HOST', postgres.secret, 'host') + .withSecret('POSTGRES_PORT', postgres.secret, 'port') + .withSecret('POSTGRES_USER', postgres.secret, 'user') + .withSecret('POSTGRES_PASSWORD', postgres.secret, 'password') + .withSecret('POSTGRES_DB', postgres.secret, 'database') + .withSecret('POSTGRES_SSL', postgres.secret, 'ssl') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/redis.ts b/deployment/services/redis.ts index 2f9b7f600..ae2fb4762 100644 --- a/deployment/services/redis.ts +++ b/deployment/services/redis.ts @@ -1,19 +1,25 @@ import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { serviceLocalHost } from '../utils/local-endpoint'; import { Redis as RedisStore } from '../utils/redis'; +import { ServiceSecret } from '../utils/secrets'; +import { Environment } from './environment'; const redisConfig = new pulumi.Config('redis'); +export class RedisSecret extends ServiceSecret<{ + password: string | pulumi.Output; + host: string | pulumi.Output; + port: string | pulumi.Output; +}> {} + export type Redis = ReturnType; -export function deployRedis({ deploymentEnv }: { deploymentEnv: DeploymentEnvironment }) { - const redisPassword = redisConfig.require('password'); +export function deployRedis(input: { environment: Environment }) { + const redisPassword = redisConfig.requireSecret('password'); const redisApi = new RedisStore({ password: redisPassword, }).deploy({ - limits: isProduction(deploymentEnv) + limits: input.environment.isProduction ? { memory: '800Mi', cpu: '1000m', @@ -24,13 +30,17 @@ export function deployRedis({ deploymentEnv }: { deploymentEnv: DeploymentEnviro }, }); + const host = serviceLocalHost(redisApi.service); + const port = String(redisApi.port); + const secret = new RedisSecret('redis', { + password: redisConfig.requireSecret('password'), + host, + port, + }); + return { deployment: redisApi.deployment, service: redisApi.service, - config: { - host: serviceLocalHost(redisApi.service), - port: redisApi.port, - password: redisPassword, - }, + secret, }; } diff --git a/deployment/services/s3.ts b/deployment/services/s3.ts new file mode 100644 index 000000000..1b1d8eaa5 --- /dev/null +++ b/deployment/services/s3.ts @@ -0,0 +1,24 @@ +import * as pulumi from '@pulumi/pulumi'; +import { ServiceSecret } from '../utils/secrets'; + +export class S3Secret extends ServiceSecret<{ + accessKeyId: string | pulumi.Output; + secretAccessKey: string | pulumi.Output; + endpoint: string | pulumi.Output; + bucket: string | pulumi.Output; +}> {} + +export function deployS3() { + const r2Config = new pulumi.Config('r2'); + + const secret = new S3Secret('cloudflare-r2', { + endpoint: r2Config.require('endpoint'), + bucket: r2Config.require('bucketName'), + accessKeyId: r2Config.requireSecret('accessKeyId'), + secretAccessKey: r2Config.requireSecret('secretAccessKey'), + }); + + return { secret }; +} + +export type S3 = ReturnType; diff --git a/deployment/services/schema.ts b/deployment/services/schema.ts index 48a37c0cf..5b652a7cc 100644 --- a/deployment/services/schema.ts +++ b/deployment/services/schema.ts @@ -1,49 +1,38 @@ -import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { ServiceDeployment } from '../utils/service-deployment'; import type { Broker } from './cf-broker'; +import { Docker } from './docker'; +import { Environment } from './environment'; import { Redis } from './redis'; - -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); +import { Sentry } from './sentry'; export type Schema = ReturnType; export function deploySchema({ - deploymentEnv, + environment, redis, broker, - release, image, - imagePullSecret, + docker, + sentry, }: { image: string; - release: string; - deploymentEnv: DeploymentEnvironment; + environment: Environment; redis: Redis; broker: Broker; - imagePullSecret: k8s.core.v1.Secret; + docker: Docker; + sentry: Sentry; }) { return new ServiceDeployment( 'schema-service', { image, - imagePullSecret, + imagePullSecret: docker.secret, availabilityOnEveryNode: true, env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - RELEASE: release, - REDIS_HOST: redis.config.host, - REDIS_PORT: String(redis.config.port), - REDIS_PASSWORD: redis.config.password, - ENCRYPTION_SECRET: commonConfig.requireSecret('encryptionSecret'), + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', REQUEST_BROKER: '1', - REQUEST_BROKER_ENDPOINT: broker.workerBaseUrl, - REQUEST_BROKER_SIGNATURE: broker.secretSignature, SCHEMA_CACHE_POLL_INTERVAL_MS: '150', SCHEMA_CACHE_TTL_MS: '65000' /* 65s */, SCHEMA_CACHE_SUCCESS_TTL_MS: '86400000' /* 24h */, @@ -53,9 +42,17 @@ export function deploySchema({ livenessProbe: '/_health', startupProbe: '/_health', exposesMetrics: true, - replicas: isProduction(deploymentEnv) ? 3 : 1, + replicas: environment.isProduction ? 3 : 1, pdb: true, }, [redis.deployment, redis.service], - ).deploy(); + ) + .withSecret('REDIS_HOST', redis.secret, 'host') + .withSecret('REDIS_PORT', redis.secret, 'port') + .withSecret('REDIS_PASSWORD', redis.secret, 'password') + .withSecret('REQUEST_BROKER_ENDPOINT', broker.secret, 'baseUrl') + .withSecret('REQUEST_BROKER_SIGNATURE', broker.secret, 'secretSignature') + .withSecret('ENCRYPTION_SECRET', environment.encryptionSecret, 'encryptionPrivateKey') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/sentry-events.ts b/deployment/services/sentry-events.ts index 6b21b7243..6f77508b4 100644 --- a/deployment/services/sentry-events.ts +++ b/deployment/services/sentry-events.ts @@ -1,16 +1,17 @@ import * as k8s from '@pulumi/kubernetes'; -import * as pulumi from '@pulumi/pulumi'; import { ServiceDeployment } from '../utils/service-deployment'; +import { Docker } from './docker'; +import { Environment } from './environment'; +import { Sentry } from './sentry'; export function deploySentryEventsMonitor(config: { - envName: string; - imagePullSecret: k8s.core.v1.Secret; + environment: Environment; + docker: Docker; + sentry: Sentry; }) { const namepsacesToTrack = ['default', 'observability', 'contour', 'cert-manager']; - const commonConfig = new pulumi.Config('common'); - const commonEnv = commonConfig.requireObject>('env'); - if (commonEnv.SENTRY_DSN) { + if (config.sentry.enabled && config.sentry.secret) { let serviceAccount = new k8s.core.v1.ServiceAccount('sentry-k8s-agent', { metadata: { name: 'sentry-k8s-agent', @@ -71,14 +72,16 @@ export function deploySentryEventsMonitor(config: { return new ServiceDeployment('sentry-events-monitor', { image: 'ghcr.io/the-guild-org/sentry-kubernetes:d9f489ced1a8eeff4a08c7e2bce427a1545dbbbd', - imagePullSecret: config.imagePullSecret, + imagePullSecret: config.docker.secret, serviceAccountName: serviceAccount.metadata.name, env: { - SENTRY_DSN: commonEnv.SENTRY_DSN, - SENTRY_ENVIRONMENT: config.envName === 'dev' ? 'development' : config.envName, + SENTRY_ENVIRONMENT: + config.environment.envName === 'dev' ? 'development' : config.environment.envName, SENTRY_K8S_WATCH_NAMESPACES: namepsacesToTrack.join(','), }, - }).deploy(); + }) + .withSecret('SENTRY_DSN', config.sentry.secret, 'dsn') + .deploy(); } else { console.warn('SENTRY_DSN is not set, skipping Sentry events monitor deployment'); diff --git a/deployment/services/sentry.ts b/deployment/services/sentry.ts new file mode 100644 index 000000000..b8b65b243 --- /dev/null +++ b/deployment/services/sentry.ts @@ -0,0 +1,29 @@ +import { Config, Output } from '@pulumi/pulumi'; +import { ServiceSecret } from '../utils/secrets'; + +export class SentrySecret extends ServiceSecret<{ + dsn: string | Output; +}> {} + +export function configureSentry() { + const sentryConfig = new Config('sentry'); + const isEnabled = sentryConfig.requireBoolean('enabled'); + + if (isEnabled) { + const secret = new SentrySecret('sentry', { + dsn: sentryConfig.requireSecret('dsn'), + }); + + return { + enabled: true, + secret, + }; + } + + return { + enabled: false, + secret: null, + }; +} + +export type Sentry = ReturnType; diff --git a/deployment/services/slack-app.ts b/deployment/services/slack-app.ts new file mode 100644 index 000000000..55b354153 --- /dev/null +++ b/deployment/services/slack-app.ts @@ -0,0 +1,22 @@ +import { Config, Output } from '@pulumi/pulumi'; +import { ServiceSecret } from '../utils/secrets'; + +class SlackIntegrationSecret extends ServiceSecret<{ + clientId: string | Output; + clientSecret: string | Output; +}> {} + +export function configureSlackApp() { + const slackConfig = new Config('slack'); + + const secret = new SlackIntegrationSecret('slack-app', { + clientId: slackConfig.require('clientId'), + clientSecret: slackConfig.requireSecret('clientSecret'), + }); + + return { + secret, + }; +} + +export type SlackApp = ReturnType; diff --git a/deployment/services/supertokens.ts b/deployment/services/supertokens.ts index 746e40253..afd38103a 100644 --- a/deployment/services/supertokens.ts +++ b/deployment/services/supertokens.ts @@ -1,18 +1,30 @@ import * as kx from '@pulumi/kubernetesx'; import * as pulumi from '@pulumi/pulumi'; -import { Output } from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; +import * as random from '@pulumi/random'; import { serviceLocalEndpoint } from '../utils/local-endpoint'; +import { ServiceSecret } from '../utils/secrets'; +import { Environment } from './environment'; +import { Postgres } from './postgres'; + +export class SupertokensSecret extends ServiceSecret<{ + apiKey: string | pulumi.Output; +}> {} export function deploySuperTokens( - { apiKey }: { apiKey: Output }, + postgres: Postgres, resourceOptions: { dependencies: pulumi.Resource[]; }, - deploymentEnv: DeploymentEnvironment, + environment: Environment, ) { - const apiConfig = new pulumi.Config('api'); + const supertokensApiKey = new random.RandomPassword('supertokens-api-key', { + length: 31, + special: false, + }).result; + + const secret = new SupertokensSecret('supertokens', { + apiKey: supertokensApiKey, + }); const port = 3567; const pb = new kx.PodBuilder({ @@ -56,10 +68,18 @@ export function deploySuperTokens( }, env: { POSTGRESQL_TABLE_NAMES_PREFIX: 'supertokens', - POSTGRESQL_CONNECTION_URI: apiConfig - .requireSecret('postgresConnectionString') - .apply(str => str.replace('postgres://', 'postgresql://')), - API_KEYS: apiKey, + POSTGRESQL_CONNECTION_URI: { + secretKeyRef: { + name: postgres.secret.record.metadata.name, + key: 'connectionStringPostgresql', + }, + }, + API_KEYS: { + secretKeyRef: { + name: secret.record.metadata.name, + key: 'apiKey', + }, + }, }, }, ], @@ -68,7 +88,7 @@ export function deploySuperTokens( const deployment = new kx.Deployment( 'supertokens', { - spec: pb.asDeploymentSpec({ replicas: isProduction(deploymentEnv) ? 3 : 1 }), + spec: pb.asDeploymentSpec({ replicas: environment.isProduction ? 3 : 1 }), }, { dependsOn: resourceOptions.dependencies, @@ -81,5 +101,8 @@ export function deploySuperTokens( deployment, service, localEndpoint: serviceLocalEndpoint(service), + secret, }; } + +export type Supertokens = ReturnType; diff --git a/deployment/services/tokens.ts b/deployment/services/tokens.ts index cefda0a01..c1ea65c3e 100644 --- a/deployment/services/tokens.ts +++ b/deployment/services/tokens.ts @@ -1,68 +1,60 @@ -import { parse } from 'pg-connection-string'; -import * as k8s from '@pulumi/kubernetes'; -import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { ServiceDeployment } from '../utils/service-deployment'; import { DbMigrations } from './db-migrations'; +import { Docker } from './docker'; +import { Environment } from './environment'; +import { Postgres } from './postgres'; import { Redis } from './redis'; - -const commonConfig = new pulumi.Config('common'); -const apiConfig = new pulumi.Config('api'); -const commonEnv = commonConfig.requireObject>('env'); +import { Sentry } from './sentry'; export type Tokens = ReturnType; export function deployTokens({ - deploymentEnv, + environment, dbMigrations, - redis, heartbeat, image, - release, - imagePullSecret, + docker, + postgres, + redis, + sentry, }: { image: string; - release: string; - deploymentEnv: DeploymentEnvironment; + environment: Environment; dbMigrations: DbMigrations; - redis: Redis; heartbeat?: string; - imagePullSecret: k8s.core.v1.Secret; + docker: Docker; + redis: Redis; + postgres: Postgres; + sentry: Sentry; }) { - const rawConnectionString = apiConfig.requireSecret('postgresConnectionString'); - const connectionString = rawConnectionString.apply(rawConnectionString => - parse(rawConnectionString), - ); - return new ServiceDeployment( 'tokens-service', { - imagePullSecret, + imagePullSecret: docker.secret, readinessProbe: '/_readiness', livenessProbe: '/_health', startupProbe: '/_health', exposesMetrics: true, availabilityOnEveryNode: true, - replicas: isProduction(deploymentEnv) ? 3 : 1, + replicas: environment.isProduction ? 3 : 1, image, env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - POSTGRES_HOST: connectionString.apply(connection => connection.host ?? ''), - POSTGRES_PORT: connectionString.apply(connection => connection.port || '5432'), - POSTGRES_PASSWORD: connectionString.apply(connection => connection.password ?? ''), - POSTGRES_USER: connectionString.apply(connection => connection.user ?? ''), - POSTGRES_DB: connectionString.apply(connection => connection.database ?? ''), - POSTGRES_SSL: connectionString.apply(connection => (connection.ssl ? '1' : '0')), - REDIS_HOST: redis.config.host, - REDIS_PORT: String(redis.config.port), - REDIS_PASSWORD: redis.config.password, - RELEASE: release, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', HEARTBEAT_ENDPOINT: heartbeat ?? '', }, }, [dbMigrations], - ).deploy(); + ) + .withSecret('POSTGRES_HOST', postgres.secret, 'host') + .withSecret('POSTGRES_PORT', postgres.secret, 'port') + .withSecret('POSTGRES_USER', postgres.secret, 'user') + .withSecret('POSTGRES_PASSWORD', postgres.secret, 'password') + .withSecret('POSTGRES_DB', postgres.secret, 'database') + .withSecret('POSTGRES_SSL', postgres.secret, 'ssl') + .withSecret('REDIS_HOST', redis.secret, 'host') + .withSecret('REDIS_PORT', redis.secret, 'port') + .withSecret('REDIS_PASSWORD', redis.secret, 'password') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/usage-estimation.ts b/deployment/services/usage-estimation.ts index 713b58737..4e2f2c1f8 100644 --- a/deployment/services/usage-estimation.ts +++ b/deployment/services/usage-estimation.ts @@ -1,54 +1,50 @@ -import * as k8s from '@pulumi/kubernetes'; -import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { ServiceDeployment } from '../utils/service-deployment'; import { Clickhouse } from './clickhouse'; import { DbMigrations } from './db-migrations'; - -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); +import { Docker } from './docker'; +import { Environment } from './environment'; +import { Sentry } from './sentry'; export type UsageEstimator = ReturnType; export function deployUsageEstimation({ image, - imagePullSecret, - release, - deploymentEnv, + docker, + environment, clickhouse, dbMigrations, + sentry, }: { image: string; - imagePullSecret: k8s.core.v1.Secret; - release: string; - deploymentEnv: DeploymentEnvironment; + docker: Docker; + environment: Environment; clickhouse: Clickhouse; dbMigrations: DbMigrations; + sentry: Sentry; }) { return new ServiceDeployment( 'usage-estimator', { image, - imagePullSecret, - replicas: isProduction(deploymentEnv) ? 3 : 1, + imagePullSecret: docker.secret, + replicas: environment.isProduction ? 3 : 1, readinessProbe: '/_readiness', livenessProbe: '/_health', startupProbe: '/_health', env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - CLICKHOUSE_PROTOCOL: clickhouse.config.protocol, - CLICKHOUSE_HOST: clickhouse.config.host, - CLICKHOUSE_PORT: clickhouse.config.port, - CLICKHOUSE_USERNAME: clickhouse.config.username, - CLICKHOUSE_PASSWORD: clickhouse.config.password, - RELEASE: release, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', }, exposesMetrics: true, port: 4000, }, [dbMigrations], - ).deploy(); + ) + .withSecret('CLICKHOUSE_HOST', clickhouse.secret, 'host') + .withSecret('CLICKHOUSE_PORT', clickhouse.secret, 'port') + .withSecret('CLICKHOUSE_USERNAME', clickhouse.secret, 'username') + .withSecret('CLICKHOUSE_PASSWORD', clickhouse.secret, 'password') + .withSecret('CLICKHOUSE_PROTOCOL', clickhouse.secret, 'protocol') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/usage-ingestor.ts b/deployment/services/usage-ingestor.ts index faad7fa4e..13d0d2441 100644 --- a/deployment/services/usage-ingestor.ts +++ b/deployment/services/usage-ingestor.ts @@ -1,57 +1,43 @@ -import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction, isStaging } from '../utils/helpers'; import { ServiceDeployment } from '../utils/service-deployment'; import { Clickhouse } from './clickhouse'; import { DbMigrations } from './db-migrations'; +import { Docker } from './docker'; +import { Environment } from './environment'; import { Kafka } from './kafka'; - -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); - -const clickHouseConfig = new pulumi.Config('clickhouse'); +import { Sentry } from './sentry'; export type UsageIngestor = ReturnType; export function deployUsageIngestor({ - deploymentEnv, + environment, clickhouse, kafka, dbMigrations, heartbeat, image, - release, - imagePullSecret, + docker, + sentry, }: { image: string; - release: string; - deploymentEnv: DeploymentEnvironment; + environment: Environment; clickhouse: Clickhouse; kafka: Kafka; dbMigrations: DbMigrations; heartbeat?: string; - imagePullSecret: k8s.core.v1.Secret; + docker: Docker; + sentry: Sentry; }) { + const clickHouseConfig = new pulumi.Config('clickhouse'); const numberOfPartitions = 16; - const replicas = isProduction(deploymentEnv) ? 6 : 1; - const cpuLimit = isProduction(deploymentEnv) ? '600m' : '300m'; - const maxReplicas = isProduction(deploymentEnv) ? numberOfPartitions : 2; - - const clickhouseEnv = { - CLICKHOUSE_PROTOCOL: clickhouse.config.protocol, - CLICKHOUSE_HOST: clickhouse.config.host, - CLICKHOUSE_PORT: clickhouse.config.port, - CLICKHOUSE_USERNAME: clickhouse.config.username, - CLICKHOUSE_PASSWORD: clickhouse.config.password, - CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: '30000', // flush data after max 30 seconds - CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: '200000000', // flush data when the buffer reaches 200MB - }; + const replicas = environment.isProduction ? 6 : 1; + const cpuLimit = environment.isProduction ? '600m' : '300m'; + const maxReplicas = environment.isProduction ? numberOfPartitions : 2; // Require migrationV2DataIngestionStartDate only in production and staging // Remove it once we are done with migration. const clickHouseMigrationV2DataIngestionStartDate = - isProduction(deploymentEnv) || isStaging(deploymentEnv) + environment.isProduction || environment.isStaging ? clickHouseConfig.require('migrationV2DataIngestionStartDate') : ''; @@ -59,21 +45,20 @@ export function deployUsageIngestor({ 'usage-ingestor-service', { image, - imagePullSecret, + imagePullSecret: docker.secret, replicas, readinessProbe: '/_readiness', livenessProbe: '/_health', availabilityOnEveryNode: true, env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - ...clickhouseEnv, - ...kafka.connectionEnv, - KAFKA_BROKER: kafka.config.endpoint, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', + CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS: '30000', // flush data after max 30 seconds + CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE: '200000000', // flush data when the buffer reaches 200MB + KAFKA_SASL_MECHANISM: kafka.config.saslMechanism, + KAFKA_CONCURRENCY: kafka.config.concurrency, KAFKA_TOPIC: kafka.config.topic, KAFKA_CONSUMER_GROUP: kafka.config.consumerGroup, - RELEASE: release, HEARTBEAT_ENDPOINT: heartbeat ?? '', MIGRATION_V2_INGEST_AFTER_UTC: clickHouseMigrationV2DataIngestionStartDate, }, @@ -88,12 +73,17 @@ export function deployUsageIngestor({ maxReplicas, }, }, - [ - clickhouse.deployment, - clickhouse.service, - dbMigrations, - kafka.deployment, - kafka.service, - ].filter(Boolean), - ).deploy(); + [clickhouse.deployment, clickhouse.service, dbMigrations].filter(Boolean), + ) + .withSecret('CLICKHOUSE_HOST', clickhouse.secret, 'host') + .withSecret('CLICKHOUSE_PORT', clickhouse.secret, 'port') + .withSecret('CLICKHOUSE_USERNAME', clickhouse.secret, 'username') + .withSecret('CLICKHOUSE_PASSWORD', clickhouse.secret, 'password') + .withSecret('CLICKHOUSE_PROTOCOL', clickhouse.secret, 'protocol') + .withSecret('KAFKA_SASL_USERNAME', kafka.secret, 'saslUsername') + .withSecret('KAFKA_SASL_PASSWORD', kafka.secret, 'saslPassword') + .withSecret('KAFKA_SSL', kafka.secret, 'ssl') + .withSecret('KAFKA_BROKER', kafka.secret, 'endpoint') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/usage.ts b/deployment/services/usage.ts index 18cea4d0c..d68833340 100644 --- a/deployment/services/usage.ts +++ b/deployment/services/usage.ts @@ -1,41 +1,38 @@ -import * as k8s from '@pulumi/kubernetes'; import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { serviceLocalEndpoint } from '../utils/local-endpoint'; import { ServiceDeployment } from '../utils/service-deployment'; import { DbMigrations } from './db-migrations'; +import { Docker } from './docker'; +import { Environment } from './environment'; import { Kafka } from './kafka'; import { RateLimitService } from './rate-limit'; +import { Sentry } from './sentry'; import { Tokens } from './tokens'; -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); - export type Usage = ReturnType; export function deployUsage({ - deploymentEnv, + environment, tokens, kafka, dbMigrations, rateLimit, image, - release, - imagePullSecret, + docker, + sentry, }: { image: string; - release: string; - deploymentEnv: DeploymentEnvironment; + environment: Environment; tokens: Tokens; kafka: Kafka; dbMigrations: DbMigrations; rateLimit: RateLimitService; - imagePullSecret: k8s.core.v1.Secret; + docker: Docker; + sentry: Sentry; }) { - const replicas = isProduction(deploymentEnv) ? 3 : 1; - const cpuLimit = isProduction(deploymentEnv) ? '600m' : '300m'; - const maxReplicas = isProduction(deploymentEnv) ? 6 : 2; + const replicas = environment.isProduction ? 3 : 1; + const cpuLimit = environment.isProduction ? '600m' : '300m'; + const maxReplicas = environment.isProduction ? 6 : 2; const kafkaBufferDynamic = kafka.config.bufferDynamic === 'true' || kafka.config.bufferDynamic === '1' ? '1' : '0'; @@ -43,24 +40,22 @@ export function deployUsage({ 'usage-service', { image, - imagePullSecret, + imagePullSecret: docker.secret, replicas, readinessProbe: '/_readiness', livenessProbe: '/_health', startupProbe: '/_health', availabilityOnEveryNode: true, env: { - ...deploymentEnv, - ...commonEnv, - ...kafka.connectionEnv, - SENTRY: commonEnv.SENTRY_ENABLED, - REQUEST_LOGGING: '0', // disabled - KAFKA_BROKER: kafka.config.endpoint, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', + REQUEST_LOGGING: '0', KAFKA_BUFFER_SIZE: kafka.config.bufferSize, + KAFKA_SASL_MECHANISM: kafka.config.saslMechanism, + KAFKA_CONCURRENCY: kafka.config.concurrency, KAFKA_BUFFER_INTERVAL: kafka.config.bufferInterval, KAFKA_BUFFER_DYNAMIC: kafkaBufferDynamic, KAFKA_TOPIC: kafka.config.topic, - RELEASE: release, TOKENS_ENDPOINT: serviceLocalEndpoint(tokens.service), RATE_LIMIT_ENDPOINT: serviceLocalEndpoint(rateLimit.service), }, @@ -81,8 +76,12 @@ export function deployUsage({ tokens.service, rateLimit.deployment, rateLimit.service, - kafka.deployment, - kafka.service, ].filter(Boolean), - ).deploy(); + ) + .withSecret('KAFKA_SASL_USERNAME', kafka.secret, 'saslUsername') + .withSecret('KAFKA_SASL_PASSWORD', kafka.secret, 'saslPassword') + .withSecret('KAFKA_SSL', kafka.secret, 'ssl') + .withSecret('KAFKA_BROKER', kafka.secret, 'endpoint') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/webhooks.ts b/deployment/services/webhooks.ts index cdee0a31c..b66f39cdf 100644 --- a/deployment/services/webhooks.ts +++ b/deployment/services/webhooks.ts @@ -1,57 +1,53 @@ -import * as k8s from '@pulumi/kubernetes'; -import * as pulumi from '@pulumi/pulumi'; -import { DeploymentEnvironment } from '../types'; -import { isProduction } from '../utils/helpers'; import { ServiceDeployment } from '../utils/service-deployment'; import type { Broker } from './cf-broker'; +import { Docker } from './docker'; +import { Environment } from './environment'; import { Redis } from './redis'; - -const commonConfig = new pulumi.Config('common'); -const commonEnv = commonConfig.requireObject>('env'); +import { Sentry } from './sentry'; export type Webhooks = ReturnType; export function deployWebhooks({ - deploymentEnv, - redis, + environment, heartbeat, broker, image, - release, - imagePullSecret, + docker, + redis, + sentry, }: { image: string; - release: string; - deploymentEnv: DeploymentEnvironment; - redis: Redis; - broker: Broker; + environment: Environment; heartbeat?: string; - imagePullSecret: k8s.core.v1.Secret; + docker: Docker; + broker: Broker; + redis: Redis; + sentry: Sentry; }) { return new ServiceDeployment( 'webhooks-service', { - imagePullSecret, + imagePullSecret: docker.secret, env: { - ...deploymentEnv, - ...commonEnv, - SENTRY: commonEnv.SENTRY_ENABLED, + ...environment.envVars, + SENTRY: sentry.enabled ? '1' : '0', HEARTBEAT_ENDPOINT: heartbeat ?? '', - RELEASE: release, - REDIS_HOST: redis.config.host, - REDIS_PORT: String(redis.config.port), - REDIS_PASSWORD: redis.config.password, REQUEST_BROKER: '1', - REQUEST_BROKER_ENDPOINT: broker.workerBaseUrl, - REQUEST_BROKER_SIGNATURE: broker.secretSignature, }, readinessProbe: '/_readiness', livenessProbe: '/_health', startupProbe: '/_health', exposesMetrics: true, - replicas: isProduction(deploymentEnv) ? 3 : 1, + replicas: environment.isProduction ? 3 : 1, image, }, [redis.deployment, redis.service], - ).deploy(); + ) + .withSecret('REDIS_HOST', redis.secret, 'host') + .withSecret('REDIS_PORT', redis.secret, 'port') + .withSecret('REDIS_PASSWORD', redis.secret, 'password') + .withSecret('REQUEST_BROKER_ENDPOINT', broker.secret, 'baseUrl') + .withSecret('REQUEST_BROKER_SIGNATURE', broker.secret, 'secretSignature') + .withConditionalSecret(sentry.enabled, 'SENTRY_DSN', sentry.secret, 'dsn') + .deploy(); } diff --git a/deployment/services/zendesk.ts b/deployment/services/zendesk.ts new file mode 100644 index 000000000..164a393fd --- /dev/null +++ b/deployment/services/zendesk.ts @@ -0,0 +1,33 @@ +import { Config, Output } from '@pulumi/pulumi'; +import { ServiceSecret } from '../utils/secrets'; +import { Environment } from './environment'; + +export class ZendeskSecret extends ServiceSecret<{ + subdomain: string | Output; + username: string | Output; + password: string | Output; +}> {} + +export function configureZendesk(input: { environment: Environment }) { + if (!input.environment.isProduction) { + return { + enabled: false, + secret: null, + }; + } + + const zendeskConfig = new Config('zendesk'); + + const secret = new ZendeskSecret('zendesk', { + subdomain: zendeskConfig.require('subdomain'), + username: zendeskConfig.require('username'), + password: zendeskConfig.requireSecret('password'), + }); + + return { + enabled: true, + secret, + }; +} + +export type Zendesk = ReturnType; diff --git a/deployment/types.ts b/deployment/types.ts index c830c6310..49a550412 100644 --- a/deployment/types.ts +++ b/deployment/types.ts @@ -1,11 +1,5 @@ import * as pulumi from '@pulumi/pulumi'; -export interface DeploymentEnvironment { - ENVIRONMENT: string; - NODE_ENV: string; - DEPLOYED_DNS: string; -} - export interface RegistryConfig { registry: string; registryToken: pulumi.Output; diff --git a/deployment/utils/cloudflare.ts b/deployment/utils/cloudflare.ts index 2089d3e0c..7b5d79853 100644 --- a/deployment/utils/cloudflare.ts +++ b/deployment/utils/cloudflare.ts @@ -2,6 +2,7 @@ import { readFileSync } from 'fs'; import { resolve } from 'path'; import * as cf from '@pulumi/cloudflare'; import * as pulumi from '@pulumi/pulumi'; +import { S3 } from '../services/s3'; export class CloudflareCDN { constructor( @@ -9,14 +10,9 @@ export class CloudflareCDN { envName: string; zoneId: string; cdnDnsRecord: string; - sentryDsn: string; + sentryDsn: string | pulumi.Output; release: string; - s3Config: { - endpoint: string; - bucketName: string; - accessKeyId: pulumi.Output; - secretAccessKey: pulumi.Output; - }; + s3: S3; }, ) {} @@ -78,19 +74,19 @@ export class CloudflareCDN { }, { name: 'S3_ENDPOINT', - text: this.config.s3Config.endpoint, + text: this.config.s3.secret.raw.endpoint, }, { name: 'S3_ACCESS_KEY_ID', - text: this.config.s3Config.accessKeyId, + text: this.config.s3.secret.raw.accessKeyId, }, { name: 'S3_SECRET_ACCESS_KEY', - text: this.config.s3Config.secretAccessKey, + text: this.config.s3.secret.raw.secretAccessKey, }, { name: 'S3_BUCKET_NAME', - text: this.config.s3Config.bucketName, + text: this.config.s3.secret.raw.bucket, }, ], }); @@ -118,7 +114,7 @@ export class CloudflareBroker { zoneId: string; cdnDnsRecord: string; secretSignature: pulumi.Output; - sentryDsn: string; + sentryDsn: string | pulumi.Output; release: string; loki: null | { endpoint: string; diff --git a/deployment/utils/helpers.ts b/deployment/utils/helpers.ts index b6cfbc066..354b3e430 100644 --- a/deployment/utils/helpers.ts +++ b/deployment/utils/helpers.ts @@ -1,21 +1,3 @@ -import { DeploymentEnvironment } from '../types'; - -export function isProduction(deploymentEnv: DeploymentEnvironment | string): boolean { - return isDeploymentEnvironment(deploymentEnv) - ? deploymentEnv.ENVIRONMENT === 'production' || deploymentEnv.ENVIRONMENT === 'prod' - : deploymentEnv === 'production' || deploymentEnv === 'prod'; -} - -export function isStaging(deploymentEnv: DeploymentEnvironment | string): boolean { - return isDeploymentEnvironment(deploymentEnv) - ? deploymentEnv.ENVIRONMENT === 'staging' - : deploymentEnv === 'staging'; -} - -export function isDeploymentEnvironment(value: any): value is DeploymentEnvironment { - return value && typeof value === 'object' && typeof value['ENVIRONMENT'] === 'string'; -} - export function isDefined(value: T | null | undefined): value is T { return value !== null && value !== undefined; } diff --git a/deployment/utils/redis.ts b/deployment/utils/redis.ts index 1d5840d56..3020a7354 100644 --- a/deployment/utils/redis.ts +++ b/deployment/utils/redis.ts @@ -1,5 +1,6 @@ import * as k8s from '@pulumi/kubernetes'; import * as kx from '@pulumi/kubernetesx'; +import { Output } from '@pulumi/pulumi'; import { getLocalComposeConfig } from './local-config'; import { normalizeEnv, PodBuilder } from './pod-builder'; @@ -9,7 +10,7 @@ export class Redis { constructor( protected options: { env?: kx.types.Container['env']; - password: string; + password: Output; }, ) {} @@ -17,7 +18,7 @@ export class Redis { const redisService = getLocalComposeConfig().service('redis'); const name = 'redis-store'; - const env = normalizeEnv(this.options.env ?? {}).concat([ + const env: k8s.types.input.core.v1.EnvVar[] = normalizeEnv(this.options.env ?? {}).concat([ { name: 'REDIS_PASSWORD', value: this.options.password, @@ -30,24 +31,28 @@ export class Redis { }, }, }, - ]); + ] satisfies k8s.types.input.core.v1.EnvVar[]); const cm = new kx.ConfigMap('redis-scripts', { data: { - 'readiness.sh': `#!/bin/bash -response=$(timeout -s SIGTERM 3 $1 redis-cli -h localhost -a ${this.options.password} -p ${PORT} ping) -if [ "$response" != "PONG" ]; then - echo "$response" - exit 1 -fi - `, - 'liveness.sh': `#!/bin/bash -response=$(timeout -s SIGTERM 3 $1 redis-cli -h localhost -a ${this.options.password} -p ${PORT} ping) -if [ "$response" != "PONG" ] && [ "$response" != "LOADING Redis is loading the dataset in memory" ]; then - echo "$response" - exit 1 -fi - `, + 'readiness.sh': this.options.password.apply( + p => `#!/bin/bash + response=$(timeout -s SIGTERM 3 $1 redis-cli -h localhost -a ${p} -p ${PORT} ping) + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi + `, + ), + 'liveness.sh': this.options.password.apply( + p => `#!/bin/bash + response=$(timeout -s SIGTERM 3 $1 redis-cli -h localhost -a ${p} -p ${PORT} ping) + if [ "$response" != "PONG" ] && [ "$response" != "LOADING Redis is loading the dataset in memory" ]; then + echo "$response" + exit 1 + fi + `, + ), }, }); diff --git a/deployment/utils/secrets.ts b/deployment/utils/secrets.ts new file mode 100644 index 000000000..b7257bf97 --- /dev/null +++ b/deployment/utils/secrets.ts @@ -0,0 +1,21 @@ +import * as k8s from '@pulumi/kubernetes'; +import { Output } from '@pulumi/pulumi'; + +export class ServiceSecret>> { + public record: k8s.core.v1.Secret; + public raw: T; + + constructor( + protected name: string, + protected data: T, + ) { + this.raw = data; + this.record = new k8s.core.v1.Secret(this.name, { + metadata: { + name: this.name, + }, + stringData: this.data, + type: 'Opaque', + }); + } +} diff --git a/deployment/utils/service-deployment.ts b/deployment/utils/service-deployment.ts index 3eea0a013..aa70f3547 100644 --- a/deployment/utils/service-deployment.ts +++ b/deployment/utils/service-deployment.ts @@ -3,13 +3,35 @@ import * as kx from '@pulumi/kubernetesx'; import * as pulumi from '@pulumi/pulumi'; import { isDefined } from './helpers'; import { normalizeEnv, PodBuilder } from './pod-builder'; +import { ServiceSecret } from './secrets'; type ProbeConfig = Omit< k8s.types.input.core.v1.Probe, 'httpGet' | 'exec' | 'grpc' | 'tcpSocket' > & { endpoint: string }; +function normalizeEnvSecrets(envSecrets?: Record>) { + return envSecrets + ? Object.keys(envSecrets).map(name => ({ + name, + valueFrom: { + secretKeyRef: { + name: envSecrets[name].secret.record.metadata.name, + key: envSecrets[name].key, + }, + }, + })) + : []; +} + +export type ServiceSecretBinding> = { + secret: ServiceSecret; + key: keyof T | pulumi.Output; +}; + export class ServiceDeployment { + private envSecrets: Record> = {}; + constructor( protected name: string, protected options: { @@ -47,6 +69,29 @@ export class ServiceDeployment { protected parent?: pulumi.Resource | null, ) {} + withSecret>>( + envVar: string, + secret: ServiceSecret, + key: keyof T, + ) { + this.envSecrets[envVar] = { secret, key }; + + return this; + } + + withConditionalSecret>>( + enabled: boolean, + envVar: string, + secret: ServiceSecret | null, + key: keyof T, + ) { + if (enabled && secret) { + this.envSecrets[envVar] = { secret, key }; + } + + return this; + } + deployAsJob() { const { pb } = this.createPod(true); @@ -64,6 +109,7 @@ export class ServiceDeployment { createPod(asJob: boolean) { const port = this.options.port || 3000; const additionalEnv: any[] = normalizeEnv(this.options.env); + const secretsEnv: any[] = normalizeEnvSecrets(this.envSecrets); let startupProbe: k8s.types.input.core.v1.Probe | undefined = undefined; let livenessProbe: k8s.types.input.core.v1.Probe | undefined = undefined; @@ -183,7 +229,9 @@ export class ServiceDeployment { }, }, }, - ].concat(additionalEnv), + ] + .concat(additionalEnv) + .concat(secretsEnv), name: this.name, image: this.options.image, resources: this.options?.autoScaling?.cpu.limit