feat: external api v1 route (get logs properties/chart + metrics tags/chart) + Mongo DB migration script (#132)

1. Implement an additional API route to enable users to access data from the HyperDX API using a standard bearer token authentication method
2. Setup Mongo DB migration tool

.changeset/new-donkeys-protect.md

@@ -0,0 +1,6 @@
+---
+'@hyperdx/api': minor
+'@hyperdx/app': minor
+---
+
+feat: external api v1 route (REQUIRES db migration) + Mongo DB migration script

Makefile

@@ -48,6 +48,10 @@ dev-unit:
 ci-unit:
 	npx nx run-many -t ci:unit
 
+.PHONY: dev-migrate-db
+dev-migrate-db:
+	npx nx run @hyperdx/api:dev:migrate-db
+
 .PHONY: build-local
 build-local:
 	docker build ./docker/hostmetrics -t ${IMAGE_NAME}:${LATEST_VERSION}-hostmetrics --target prod &

README.md

@@ -96,6 +96,12 @@ can change this by updating `HYPERDX_APP_**` and `HYPERDX_API_**` variables in
 the `.env` file. After making your changes, rebuild images with
 `make build-local`.
 
+**DB Migration**
+
+You can initiate the DB migration process by executing `make dev-migrate-db`.
+This will run the migration scripts in `/packages/api/migrations` against the
+local DB.
+
 ### Hosted Cloud
 
 HyperDX is also available as a hosted cloud service at

packages/api/.eslintignore

@@ -1,3 +1,5 @@
 keys
 node_modules
 archive
+migrations
+migrate-mongo-config.ts

packages/api/migrate-mongo-config.ts

@@ -0,0 +1,32 @@
+export = {
+  mongodb: {
+    // TODO Change (or review) the url to your MongoDB:
+    url: 'mongodb://localhost:27017',
+
+    // TODO Change this to your database name:
+    databaseName: 'hyperdx',
+
+    options: {
+      // useNewUrlParser: true, // removes a deprecation warning when connecting
+      // useUnifiedTopology: true, // removes a deprecating warning when connecting
+      //   connectTimeoutMS: 3600000, // increase connection timeout to 1 hour
+      //   socketTimeoutMS: 3600000, // increase socket timeout to 1 hour
+    },
+  },
+
+  // The migrations dir, can be an relative or absolute path. Only edit this when really necessary.
+  migrationsDir: 'migrations',
+
+  // The mongodb collection where the applied changes are stored. Only edit this when really necessary.
+  changelogCollectionName: 'changelog',
+
+  // The file extension to create migrations and search for in migration dir
+  migrationFileExtension: '.ts',
+
+  // Enable the algorithm to create a checksum of the file contents and use that in the comparison to determine
+  // if the file should be run.  Requires that scripts are coded to be run multiple times.
+  useFileHash: false,
+
+  // Don't change this, unless you know what you're doing
+  moduleSystem: 'commonjs',
+};

packages/api/migrations/20231130053610-add_accessKey_field_to_user_collection.ts

@@ -0,0 +1,13 @@
+import { Db, MongoClient } from 'mongodb';
+import { v4 as uuidv4 } from 'uuid';
+
+module.exports = {
+  async up(db: Db, client: MongoClient) {
+    await db
+      .collection('users')
+      .updateMany({}, { $set: { accessKey: uuidv4() } });
+  },
+  async down(db: Db, client: MongoClient) {
+    await db.collection('users').updateMany({}, { $unset: { accessKey: '' } });
+  },
+};

packages/api/package.json

@@ -72,6 +72,8 @@
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-prettier": "^4.2.1",
     "jest": "^28.1.1",
+    "migrate-mongo": "^11.0.0",
+    "mongodb": "^6.3.0",
     "nodemon": "^2.0.20",
     "rimraf": "^4.4.1",
     "supertest": "^6.3.1",
@@ -89,6 +91,8 @@
     "lint": "eslint . --ext .ts",
     "ci:lint": "yarn lint && yarn tsc --noEmit",
     "ci:int": "jest --runInBand --ci --forceExit --coverage",
-    "dev:int": "jest --watchAll --runInBand --detectOpenHandles"
+    "dev:int": "jest --watchAll --runInBand --detectOpenHandles",
+    "dev:migrate-db-create": "ts-node node_modules/.bin/migrate-mongo create -f migrate-mongo-config.ts",
+    "dev:migrate-db": "ts-node node_modules/.bin/migrate-mongo up -f migrate-mongo-config.ts"
   }
 }

packages/api/src/api-app.ts

@@ -7,6 +7,7 @@ import session from 'express-session';
 
 import * as config from './config';
 import defaultCors from './middleware/cors';
+import externalRoutersV1 from './routers/external-api/v1';
 import passport from './utils/passport';
 import routers from './routers/api';
 import usageStats from './tasks/usageStats';
@@ -82,6 +83,14 @@ app.use('/team', routers.teamRouter);
 app.use('/webhooks', routers.webhooksRouter);
 // ---------------------------------------------------------------------
 
+// TODO: Separate external API routers from internal routers
+// ---------------------------------------------------------------------
+// ----------------------- External Routers ----------------------------
+// ---------------------------------------------------------------------
+// API v1
+app.use('/api/v1', externalRoutersV1);
+// ---------------------------------------------------------------------
+
 // error handling
 app.use(appErrorHandler);
 

packages/api/src/controllers/user.ts

@@ -2,6 +2,10 @@ import User from '@/models/user';
 
 import type { ObjectId } from '@/models';
 
+export function findUserByAccessKey(accessKey: string) {
+  return User.findOne({ accessKey });
+}
+
 export function findUserById(id: string) {
   return User.findById(id);
 }

packages/api/src/middleware/auth.ts

@@ -5,6 +5,7 @@ import { setTraceAttributes } from '@hyperdx/node-opentelemetry';
 
 import * as config from '@/config';
 import logger from '@/utils/logger';
+import { findUserByAccessKey } from '@/controllers/user';
 
 import type { UserDocument } from '@/models/user';
 
@@ -58,6 +59,30 @@ export function handleAuthError(
   res.redirect(`${config.FRONTEND_URL}/login?err=${returnErr}`);
 }
 
+export async function validateUserAccessKey(
+  req: Request,
+  res: Response,
+  next: NextFunction,
+) {
+  const authHeader = req.headers.authorization;
+  if (!authHeader) {
+    return res.sendStatus(401);
+  }
+  const key = authHeader.split('Bearer ')[1];
+  if (!key) {
+    return res.sendStatus(401);
+  }
+
+  const user = await findUserByAccessKey(key);
+  if (!user) {
+    return res.sendStatus(401);
+  }
+
+  req.user = user;
+
+  next();
+}
+
 export function isUserAuthenticated(
   req: Request,
   res: Response,

packages/api/src/models/user.ts

@@ -7,6 +7,7 @@ type ObjectId = mongoose.Types.ObjectId;
 
 export interface IUser {
   _id: ObjectId;
+  accessKey: string;
   createdAt: Date;
   email: string;
   name: string;
@@ -23,6 +24,12 @@ const UserSchema = new Schema(
       required: true,
     },
     team: { type: mongoose.Schema.Types.ObjectId, ref: 'Team' },
+    accessKey: {
+      type: String,
+      default: function genUUID() {
+        return uuidv4();
+      },
+    },
   },
   {
     timestamps: true,

packages/api/src/routers/api/root.ts

@@ -52,11 +52,19 @@ router.get('/me', isUserAuthenticated, async (req, res, next) => {
       throw new Api404Error('Request without user found');
     }
 
-    const { _id: id, team: teamId, email, name, createdAt } = req.user;
+    const {
+      _id: id,
+      accessKey,
+      createdAt,
+      email,
+      name,
+      team: teamId,
+    } = req.user;
 
     const team = await getTeam(teamId);
 
     return res.json({
+      accessKey,
       createdAt,
       email,
       id,

packages/api/src/routers/external-api/__tests__/v1.test.ts

@@ -0,0 +1,205 @@
+import * as clickhouse from '@/clickhouse';
+import {
+  clearDBCollections,
+  closeDB,
+  getLoggedInAgent,
+  getServer,
+} from '@/fixtures';
+
+describe('external api v1', () => {
+  const server = getServer();
+
+  beforeAll(async () => {
+    await server.start();
+  });
+
+  afterEach(async () => {
+    await clearDBCollections();
+    jest.clearAllMocks();
+  });
+
+  afterAll(async () => {
+    await server.closeHttpServer();
+    await closeDB();
+  });
+
+  it('GET /api/v1', async () => {
+    const { agent, user } = await getLoggedInAgent(server);
+    const resp = await agent
+      .get(`/api/v1`)
+      .set('Authorization', `Bearer ${user?.accessKey}`)
+      .expect(200);
+    expect(resp.body.version).toEqual('v1');
+    expect(resp.body.user._id).toEqual(user?._id.toString());
+  });
+
+  it('GET /api/v1/metrics/tags', async () => {
+    jest.spyOn(clickhouse, 'getMetricsTags').mockResolvedValueOnce({
+      data: [
+        {
+          name: 'system.filesystem.usage - Sum',
+          tags: [
+            {
+              device: '/dev/vda1',
+              host: 'unknown',
+              mode: 'rw',
+              mountpoint: '/etc/resolv.conf',
+              state: 'reserved',
+              type: 'ext4',
+            },
+          ],
+          data_type: 'Sum',
+        },
+      ],
+      meta: [
+        {
+          name: 'name',
+          type: 'String',
+        },
+        {
+          name: 'data_type',
+          type: 'LowCardinality(String)',
+        },
+        {
+          name: 'tags',
+          type: 'Array(Map(String, String))',
+        },
+      ],
+      rows: 1,
+    });
+    const { agent, user } = await getLoggedInAgent(server);
+    const resp = await agent
+      .get(`/api/v1/metrics/tags`)
+      .set('Authorization', `Bearer ${user?.accessKey}`)
+      .expect(200);
+
+    expect(clickhouse.getMetricsTags).toBeCalledTimes(1);
+    expect(resp.body).toEqual({
+      data: [
+        {
+          name: 'system.filesystem.usage',
+          tags: [
+            {
+              device: '/dev/vda1',
+              host: 'unknown',
+              mode: 'rw',
+              mountpoint: '/etc/resolv.conf',
+              state: 'reserved',
+              type: 'ext4',
+            },
+          ],
+          type: 'Sum',
+        },
+      ],
+      meta: [
+        {
+          name: 'name',
+          type: 'String',
+        },
+        {
+          name: 'data_type',
+          type: 'LowCardinality(String)',
+        },
+        {
+          name: 'tags',
+          type: 'Array(Map(String, String))',
+        },
+      ],
+      rows: 1,
+    });
+  });
+
+  describe('POST /api/v1/metrics/chart', () => {
+    it('should return 400 if startTime is greater than endTime', async () => {
+      const { agent, user } = await getLoggedInAgent(server);
+      await agent
+        .post(`/api/v1/metrics/chart`)
+        .set('Authorization', `Bearer ${user?.accessKey}`)
+        .send({
+          aggFn: 'max_rate',
+          endTime: 1701224193940,
+          granularity: '30 second',
+          name: 'http.server.active_requests',
+          startTime: 1701233593940,
+          type: 'Sum',
+        })
+        .expect(400);
+    });
+
+    it('suucess', async () => {
+      jest.spyOn(clickhouse, 'getMetricsChart').mockResolvedValueOnce({
+        data: [
+          {
+            ts_bucket: 1701223590,
+            data: 10,
+            group: 'http.server.active_requests',
+          },
+        ],
+        meta: [
+          {
+            name: 'ts_bucket',
+            type: 'UInt32',
+          },
+          {
+            name: 'data',
+            type: 'Float64',
+          },
+          {
+            name: 'group',
+            type: 'LowCardinality(String)',
+          },
+        ],
+        rows: 1,
+      });
+      const { agent, user, team } = await getLoggedInAgent(server);
+      const resp = await agent
+        .post(`/api/v1/metrics/chart`)
+        .set('Authorization', `Bearer ${user?.accessKey}`)
+        .send({
+          aggFn: 'max_rate',
+          endTime: 1701224193940,
+          granularity: '30 second',
+          name: 'http.server.active_requests',
+          startTime: 1701223593940,
+          type: 'Sum',
+        })
+        .expect(200);
+
+      expect(clickhouse.getMetricsChart).toHaveBeenNthCalledWith(1, {
+        aggFn: 'max_rate',
+        dataType: 'Sum',
+        endTime: 1701224193940,
+        granularity: '30 second',
+        groupBy: undefined,
+        name: 'http.server.active_requests',
+        q: undefined,
+        startTime: 1701223593940,
+        teamId: team?._id.toString(),
+      });
+      expect(resp.body).toEqual({
+        data: [
+          {
+            ts_bucket: 1701223590,
+            data: 10,
+            group: 'http.server.active_requests',
+          },
+        ],
+        meta: [
+          {
+            name: 'ts_bucket',
+            type: 'UInt32',
+          },
+          {
+            name: 'data',
+            type: 'Float64',
+          },
+          {
+            name: 'group',
+            type: 'LowCardinality(String)',
+          },
+        ],
+        rows: 1,
+      });
+    });
+  });
+});

packages/api/src/routers/external-api/v1/index.ts

@@ -0,0 +1,230 @@
+import express from 'express';
+import ms from 'ms';
+import opentelemetry, { SpanStatusCode } from '@opentelemetry/api';
+import { isNumber, parseInt } from 'lodash';
+import { validateRequest } from 'zod-express-middleware';
+import { z } from 'zod';
+
+import * as clickhouse from '@/clickhouse';
+import rateLimiter from '@/utils/rateLimiter';
+import { Api400Error, Api403Error } from '@/utils/errors';
+import { getTeam } from '@/controllers/team';
+import { validateUserAccessKey } from '@/middleware/auth';
+
+const router = express.Router();
+
+const rateLimiterKeyGenerator = (req: express.Request) => {
+  return req.headers.authorization || req.ip;
+};
+
+const getDefaultRateLimiter = () =>
+  rateLimiter({
+    windowMs: 60 * 1000, // 1 minute
+    max: 100, // Limit each IP to 100 requests per `window`
+    standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
+    legacyHeaders: false, // Disable the `X-RateLimit-*` headers
+    keyGenerator: rateLimiterKeyGenerator,
+  });
+
+router.get('/', validateUserAccessKey, (req, res, next) => {
+  res.json({
+    version: 'v1',
+    user: req.user?.toJSON(),
+  });
+});
+
+router.get(
+  '/logs/properties',
+  getDefaultRateLimiter(),
+  validateUserAccessKey,
+  async (req, res, next) => {
+    try {
+      const teamId = req.user?.team.toString();
+      if (teamId == null) {
+        throw new Api403Error('Forbidden');
+      }
+
+      const team = await getTeam(teamId);
+      if (team == null) {
+        throw new Api403Error('Forbidden');
+      }
+
+      const nowInMs = Date.now();
+      const propertyTypeMappingsModel =
+        await clickhouse.buildLogsPropertyTypeMappingsModel(
+          team.logStreamTableVersion,
+          teamId,
+          nowInMs - ms('1d'),
+          nowInMs,
+        );
+
+      const data = [...propertyTypeMappingsModel.currentPropertyTypeMappings];
+      res.json({
+        data,
+        rows: data.length,
+      });
+    } catch (e) {
+      const span = opentelemetry.trace.getActiveSpan();
+      span?.recordException(e as Error);
+      span?.setStatus({ code: SpanStatusCode.ERROR });
+      next(e);
+    }
+  },
+);
+
+router.get(
+  '/logs/chart',
+  getDefaultRateLimiter(),
+  validateRequest({
+    query: z.object({
+      aggFn: z.nativeEnum(clickhouse.AggFn),
+      endTime: z.string(),
+      field: z.string().optional(),
+      granularity: z.nativeEnum(clickhouse.Granularity).optional(),
+      groupBy: z.string().optional(),
+      q: z.string().optional(),
+      startTime: z.string(),
+    }),
+  }),
+  validateUserAccessKey,
+  async (req, res, next) => {
+    try {
+      const teamId = req.user?.team.toString();
+      const { aggFn, endTime, field, granularity, groupBy, q, startTime } =
+        req.query;
+      if (teamId == null) {
+        throw new Api403Error('Forbidden');
+      }
+      const startTimeNum = parseInt(startTime);
+      const endTimeNum = parseInt(endTime);
+      if (!isNumber(startTimeNum) || !isNumber(endTimeNum)) {
+        throw new Api400Error('startTime and endTime must be numbers');
+      }
+
+      const team = await getTeam(teamId);
+      if (team == null) {
+        throw new Api403Error('Forbidden');
+      }
+
+      const propertyTypeMappingsModel =
+        await clickhouse.buildLogsPropertyTypeMappingsModel(
+          team.logStreamTableVersion,
+          teamId,
+          startTimeNum,
+          endTimeNum,
+        );
+
+      // TODO: expose this to the frontend ?
+      const MAX_NUM_GROUPS = 20;
+
+      res.json(
+        await clickhouse.getLogsChart({
+          aggFn,
+          endTime: endTimeNum,
+          // @ts-expect-error
+          field,
+          granularity,
+          // @ts-expect-error
+          groupBy,
+          maxNumGroups: MAX_NUM_GROUPS,
+          propertyTypeMappingsModel,
+          // @ts-expect-error
+          q,
+          startTime: startTimeNum,
+          tableVersion: team.logStreamTableVersion,
+          teamId,
+        }),
+      );
+    } catch (e) {
+      const span = opentelemetry.trace.getActiveSpan();
+      span?.recordException(e as Error);
+      span?.setStatus({ code: SpanStatusCode.ERROR });
+      next(e);
+    }
+  },
+);
+
+router.get(
+  '/metrics/tags',
+  getDefaultRateLimiter(),
+  validateUserAccessKey,
+  async (req, res, next) => {
+    try {
+      const teamId = req.user?.team;
+      if (teamId == null) {
+        throw new Api403Error('Forbidden');
+      }
+      const tags = await clickhouse.getMetricsTags(teamId.toString());
+      res.json({
+        data: tags.data.map(tag => ({
+          // FIXME: unify the return type of both internal and external APIs
+          name: tag.name.split(' - ')[0], // FIXME: we want to separate name and data type into two columns
+          type: tag.data_type,
+          tags: tag.tags,
+        })),
+        meta: tags.meta,
+        rows: tags.rows,
+      });
+    } catch (e) {
+      const span = opentelemetry.trace.getActiveSpan();
+      span?.recordException(e as Error);
+      span?.setStatus({ code: SpanStatusCode.ERROR });
+      next(e);
+    }
+  },
+);
+
+router.post(
+  '/metrics/chart',
+  getDefaultRateLimiter(),
+  validateRequest({
+    body: z.object({
+      aggFn: z.nativeEnum(clickhouse.AggFn),
+      endTime: z.number().int().min(0),
+      granularity: z.nativeEnum(clickhouse.Granularity),
+      groupBy: z.string().optional(),
+      name: z.string().min(1),
+      type: z.nativeEnum(clickhouse.MetricsDataType),
+      q: z.string().optional(),
+      startTime: z.number().int().min(0),
+    }),
+  }),
+  validateUserAccessKey,
+  async (req, res, next) => {
+    try {
+      const teamId = req.user?.team;
+      const { aggFn, endTime, granularity, groupBy, name, q, startTime, type } =
+        req.body;
+
+      if (teamId == null) {
+        throw new Api403Error('Forbidden');
+      }
+
+      if (startTime > endTime) {
+        throw new Api400Error('startTime must be less than endTime');
+      }
+
+      res.json(
+        await clickhouse.getMetricsChart({
+          aggFn,
+          dataType: type,
+          endTime,
+          granularity,
+          groupBy,
+          name,
+          // @ts-expect-error
+          q,
+          startTime,
+          teamId: teamId.toString(),
+        }),
+      );
+    } catch (e) {
+      const span = opentelemetry.trace.getActiveSpan();
+      span?.recordException(e as Error);
+      span?.setStatus({ code: SpanStatusCode.ERROR });
+      next(e);
+    }
+  },
+);
+
+export default router;

packages/api/src/utils/errors.ts

@@ -1,10 +1,11 @@
 export enum StatusCode {
-  OK = 200,
   BAD_REQUEST = 400,
-  UNAUTHORIZED = 401,
+  CONFLICT = 409,
   FORBIDDEN = 403,
-  NOT_FOUND = 404,
   INTERNAL_SERVER = 500,
+  NOT_FOUND = 404,
+  OK = 200,
+  UNAUTHORIZED = 401,
 }
 
 export class BaseError extends Error {
@@ -48,6 +49,24 @@ export class Api404Error extends BaseError {
   }
 }
 
+export class Api401Error extends BaseError {
+  constructor(name: string) {
+    super(name, StatusCode.UNAUTHORIZED, true, 'Unauthorized');
+  }
+}
+
+export class Api403Error extends BaseError {
+  constructor(name: string) {
+    super(name, StatusCode.FORBIDDEN, true, 'Forbidden');
+  }
+}
+
+export class Api409Error extends BaseError {
+  constructor(name: string) {
+    super(name, StatusCode.CONFLICT, true, 'Conflict');
+  }
+}
+
 export const isOperationalError = (error: Error) => {
   if (error instanceof BaseError) {
     return error.isOperational;

packages/app/src/TeamPage.tsx

@@ -262,6 +262,22 @@ export default function TeamPage() {
                   </Modal>
                 </div>
               </div>
+              {!isLoadingMe && me != null && (
+                <div className="my-4 fs-5">
+                  <div className="text-muted">Personal API Access Key: </div>
+                  <Badge bg="primary" data-test-id="apiKey">
+                    {me.accessKey}
+                  </Badge>
+                  <CopyToClipboard text={me.accessKey}>
+                    <Button
+                      variant="link"
+                      className="px-0 text-muted-hover text-decoration-none fs-7 ms-3"
+                    >
+                      📋 Copy Key
+                    </Button>
+                  </CopyToClipboard>
+                </div>
+              )}
               <div className="my-5">
                 <h2>Slack Webhooks</h2>
                 <div className="text-muted">

yarn.lock

@@ -4766,6 +4766,13 @@
   resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz#2b8e60e33906459219aa587e9d1a612ae994cfe7"
   integrity sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==
 
+"@types/whatwg-url@^11.0.2":
+  version "11.0.3"
+  resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-11.0.3.tgz#9f584c9a9421f0971029ee504dd62a831cb8f3aa"
+  integrity sha512-z1ELvMijRL1QmU7QuzDkeYXSF2+dXI0ITKoQsIoVKcNBOiK5RMmWy+pYYxJTHFt8vkpZe7UsvRErQwcxZkjoUw==
+  dependencies:
+    "@types/webidl-conversions" "*"
+
 "@types/whatwg-url@^8.2.1":
   version "8.2.2"
   resolved "https://registry.yarnpkg.com/@types/whatwg-url/-/whatwg-url-8.2.2.tgz#749d5b3873e845897ada99be4448041d4cc39e63"
@@ -5698,6 +5705,11 @@ bson@^4.7.2:
   dependencies:
     buffer "^5.6.0"
 
+bson@^6.2.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/bson/-/bson-6.2.0.tgz#4b6acafc266ba18eeee111373c2699304a9ba0a3"
+  integrity sha512-ID1cI+7bazPDyL9wYy9GaQ8gEEohWvcUl/Yf0dIdutJxnmInEEyCsb4awy/OiBfall7zBA179Pahi3vCdFze3Q==
+
 buffer-equal-constant-time@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
@@ -5971,6 +5983,15 @@ cli-spinners@2.6.1:
   resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-2.6.1.tgz#adc954ebe281c37a6319bfa401e6dd2488ffb70d"
   integrity sha512-x/5fWmGMnbKQAaNwN+UZlV79qBLM9JFnJuJ03gIi5whrob0xV0ofNVHy9DhwGdsMJQc2OKv0oGmLzvaqvAVv+g==
 
+cli-table3@^0.6.1:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.3.tgz#61ab765aac156b52f222954ffc607a6f01dbeeb2"
+  integrity sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==
+  dependencies:
+    string-width "^4.2.0"
+  optionalDependencies:
+    "@colors/colors" "1.5.0"
+
 cli-truncate@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/cli-truncate/-/cli-truncate-2.1.0.tgz#c39e28bf05edcde5be3b98992a22deed5a2b93c7"
@@ -6130,7 +6151,7 @@ comma-separated-tokens@^2.0.0:
   resolved "https://registry.yarnpkg.com/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz#4e89c9458acb61bc8fef19f4529973b2392839ee"
   integrity sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==
 
-commander@^9.0.0, commander@^9.4.1:
+commander@^9.0.0, commander@^9.1.0, commander@^9.4.1:
   version "9.5.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-9.5.0.tgz#bc08d1eb5cedf7ccb797a96199d41c7bc3e60d30"
   integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==
@@ -8114,6 +8135,11 @@ flexsearch@^0.7.21:
   resolved "https://registry.yarnpkg.com/flexsearch/-/flexsearch-0.7.31.tgz#065d4110b95083110b9b6c762a71a77cc52e4702"
   integrity sha512-XGozTsMPYkm+6b5QL3Z9wQcJjNYxp0CYn3U1gO7dwD6PAqU1SVWZxI9CCg3z+ml3YfqdPnrBehaBrnH2AGKbNA==
 
+fn-args@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/fn-args/-/fn-args-5.0.0.tgz#7a18e105c8fb3bf0a51c30389bf16c9ebe740bb3"
+  integrity sha512-CtbfI3oFFc3nbdIoHycrfbrxiGgxXBXXuyOl49h47JawM1mYrqpiRqnH5CB2mBatdXvHHOUO6a+RiAuuvKt0lw==
+
 fn.name@1.x.x:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
@@ -8217,6 +8243,15 @@ fs-constants@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
   integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
 
+fs-extra@^10.0.1:
+  version "10.1.0"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf"
+  integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==
+  dependencies:
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^2.0.0"
+
 fs-extra@^11.1.0:
   version "11.1.0"
   resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.0.tgz#5784b102104433bb0e090f48bfc4a30742c357ed"
@@ -11595,6 +11630,19 @@ microseconds@0.2.0:
   resolved "https://registry.yarnpkg.com/microseconds/-/microseconds-0.2.0.tgz#233b25f50c62a65d861f978a4a4f8ec18797dc39"
   integrity sha512-n7DHHMjR1avBbSpsTBj6fmMGh2AGrifVV4e+WYc3Q9lO+xnSZ3NyhcBND3vzzatt05LFhoKFRxrIyklmLlUtyA==
 
+migrate-mongo@^11.0.0:
+  version "11.0.0"
+  resolved "https://registry.yarnpkg.com/migrate-mongo/-/migrate-mongo-11.0.0.tgz#d1b2291624fe8e134a0666ca77ad2fa18f42e337"
+  integrity sha512-GB/gHzUwp/fL1w6ksNGihTyb+cSrm6NbVLlz1OSkQKaLlzAXMwH7iKK2ZS7W5v+I8vXiY2rL58WTUZSAL6QR+A==
+  dependencies:
+    cli-table3 "^0.6.1"
+    commander "^9.1.0"
+    date-fns "^2.28.0"
+    fn-args "^5.0.0"
+    fs-extra "^10.0.1"
+    lodash "^4.17.21"
+    p-each-series "^2.2.0"
+
 mime-db@1.52.0, "mime-db@>= 1.43.0 < 2":
   version "1.52.0"
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
@@ -11737,6 +11785,14 @@ mongodb-connection-string-url@^2.6.0:
     "@types/whatwg-url" "^8.2.1"
     whatwg-url "^11.0.0"
 
+mongodb-connection-string-url@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.0.tgz#b4f87f92fd8593f3b9365f592515a06d304a1e9c"
+  integrity sha512-t1Vf+m1I5hC2M5RJx/7AtxgABy1cZmIPQRMXw+gEIPn/cZNF3Oiy+l0UIypUwVB5trcWHq3crg2g3uAR9aAwsQ==
+  dependencies:
+    "@types/whatwg-url" "^11.0.2"
+    whatwg-url "^13.0.0"
+
 mongodb@4.17.1:
   version "4.17.1"
   resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-4.17.1.tgz#ccff6ddbda106d5e06c25b0e4df454fd36c5f819"
@@ -11749,6 +11805,15 @@ mongodb@4.17.1:
     "@aws-sdk/credential-providers" "^3.186.0"
     "@mongodb-js/saslprep" "^1.1.0"
 
+mongodb@^6.3.0:
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-6.3.0.tgz#ec9993b19f7ed2ea715b903fcac6171c9d1d38ca"
+  integrity sha512-tt0KuGjGtLUhLoU263+xvQmPHEGTw5LbcNC73EoFRYgSHwZt5tsoJC110hDyO1kjQzpgNrpdcSza9PknWN4LrA==
+  dependencies:
+    "@mongodb-js/saslprep" "^1.1.0"
+    bson "^6.2.0"
+    mongodb-connection-string-url "^3.0.0"
+
 mongoose@^6.12.0:
   version "6.12.0"
   resolved "https://registry.yarnpkg.com/mongoose/-/mongoose-6.12.0.tgz#53035998245a029144411331373c5ce878f62815"
@@ -12360,6 +12425,11 @@ outdent@^0.5.0:
   resolved "https://registry.yarnpkg.com/outdent/-/outdent-0.5.0.tgz#9e10982fdc41492bb473ad13840d22f9655be2ff"
   integrity sha512-/jHxFIzoMXdqPzTaCpFzAAWhpkSjZPF4Vsn6jAfNpmbH/ymsmd7Qc6VE9BGn0L6YMj6uwpQLxCECpus4ukKS9Q==
 
+p-each-series@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/p-each-series/-/p-each-series-2.2.0.tgz#105ab0357ce72b202a8a8b94933672657b5e2a9a"
+  integrity sha512-ycIL2+1V32th+8scbpTvyHNaHe02z0sjgh91XXjAk+ZeXoPN4Z46DVUnzdso0aX4KckKw0FNNFHdjZ2UsZvxiA==
+
 p-filter@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/p-filter/-/p-filter-2.1.0.tgz#1b1472562ae7a0f742f0f3d3d3718ea66ff9c09c"
@@ -12980,6 +13050,11 @@ punycode@2.x.x, punycode@^2.1.0, punycode@^2.1.1:
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.0.tgz#f67fa67c94da8f4d0cfff981aee4118064199b8f"
   integrity sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==
 
+punycode@^2.3.0:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5"
+  integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==
+
 qs@6.11.0, qs@^6.11.0:
   version "6.11.0"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a"
@@ -15110,6 +15185,13 @@ tr46@^3.0.0:
   dependencies:
     punycode "^2.1.1"
 
+tr46@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/tr46/-/tr46-4.1.1.tgz#281a758dcc82aeb4fe38c7dfe4d11a395aac8469"
+  integrity sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==
+  dependencies:
+    punycode "^2.3.0"
+
 tr46@~0.0.3:
   version "0.0.3"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
@@ -15903,6 +15985,14 @@ whatwg-url@^11.0.0:
     tr46 "^3.0.0"
     webidl-conversions "^7.0.0"
 
+whatwg-url@^13.0.0:
+  version "13.0.0"
+  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-13.0.0.tgz#b7b536aca48306394a34e44bda8e99f332410f8f"
+  integrity sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==
+  dependencies:
+    tr46 "^4.1.1"
+    webidl-conversions "^7.0.0"
+
 whatwg-url@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"