first commit

.gitattributes

@@ -0,0 +1 @@
+/.yarn/releases/** binary

.github/workflows/main.yml

@@ -0,0 +1,49 @@
+name: Main
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  lint:
+    timeout-minutes: 8
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+      - name: Install root dependencies
+        uses: bahmutov/npm-install@v1
+      - name: Install core libs
+        run: sudo apt-get install --yes curl bc
+      - name: Install vector
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.vector.dev | bash -s -- -y
+          ~/.vector/bin/vector --version
+          ln -s ~/.vector/bin/vector /usr/local/bin/vector
+      - name: Run lint + type check
+        run: yarn ci:lint
+  integration:
+    timeout-minutes: 8
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+      - name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v2
+      - name: Build images
+        run: |
+          docker buildx create --use --driver=docker-container
+          docker buildx bake -f ./docker-compose.ci.yml --set *.cache-to="type=gha" --set *.cache-from="type=gha" --load
+      - name: Run integration tests
+        run: yarn ci:int

.gitignore

@@ -0,0 +1,56 @@
+# misc
+**/.DS_Store
+**/*.pem
+**/keys
+
+# logs
+**/*.log
+**/npm-debug.log*
+**/yarn-debug.log*
+**/yarn-error.log*
+**/lerna-debug.log*
+
+# dotenv environment variable files
+**/.env
+**/.env.development.local
+**/.env.test.local
+**/.env.production.local
+**/.env.local
+
+# Next.js build output
+packages/app/.next
+packages/app/.pnp
+packages/app/.pnp.js
+packages/app/.vercel
+packages/app/coverage
+packages/app/out
+
+# optional npm cache directory
+**/.npm
+
+# dependency directories
+**/node_modules
+
+# build output
+**/dist
+**/build
+**/tsconfig.tsbuildinfo
+
+# jest coverage report
+**/coverage
+
+# e2e
+e2e/cypress/screenshots/
+e2e/cypress/videos/
+e2e/cypress/results
+
+# scripts
+scripts/*.csv
+**/venv
+**/__pycache__/
+*.py[cod]
+*$py.class
+
+# docker
+docker-compose.prod.yml
+.volumes

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npx lint-staged

.nvmrc

@@ -0,0 +1 @@
+v18.15.0

.prettierignore

@@ -0,0 +1,5 @@
+# Ignore artifacts:
+dist
+coverage
+tests
+.volumes

.prettierrc

@@ -0,0 +1,13 @@
+{
+  "printWidth": 80,
+  "tabWidth": 2,
+  "useTabs": false,
+  "semi": true,
+  "singleQuote": true,
+  "trailingComma": "all",
+  "bracketSpacing": true,
+  "jsxBracketSameLine": false,
+  "fluid": false,
+  "arrowParens": "avoid",
+  "proseWrap": "always"
+}

.yarnrc

@@ -0,0 +1,5 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+yarn-path ".yarn/releases/yarn-1.22.18.cjs"

CONTRIBUTING.md

@@ -0,0 +1,31 @@
+# Contributing
+
+## Architecture Overview
+
+![architecture](./.github/images/architecture.png)
+
+Service Descriptions:
+
+- otel: OpenTelemetry Collector, allows us to receive OpenTelemetry data from
+  instrumented applications and forward it to the ingestor for futher
+  processing.
+- ingestor: Vector-based event pipeline that receives Otel and non-Otel events
+  and parses/normalizes/forwards it to the aggregator.
+- aggregator: Node.js service that receives events from the ingestor, verifies
+  authentication, and inserts it to Clickhouse for storage.
+- clickhouse: Clickhouse database, stores all events.
+- db: MongoDB, stores user/alert/dashboard data.
+- api: Node.js API, executes Clickhouse queries on behalf of the frontend.
+- app: Next.js frontend, serves the UI.
+- task-check-alerts: Checks for alert criteria and fires off any alerts as
+  needed.
+
+## Development
+
+You can get started by deploying a complete stack via Docker Compose. The core
+services are all hot-reloaded, so you can make changes to the code and see them
+reflected in real-time.
+
+If you need help getting started,
+[join our Discord](https://discord.gg/FErRRKU78j) and we're more than happy to
+get you set up!

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 DeploySentinel, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,183 @@
+<p align="center">
+  <img alt="hyperdx logo dark" src="./.github/images/logo_dark.png#gh-dark-mode-only">
+  <img alt="hyperdx logo light" src="./.github/images/logo_light.png#gh-light-mode-only">
+</p>
+
+---
+
+# HyperDX
+
+[HyperDX](https://hyperdx.io) helps engineers figure out why production is
+broken faster by centralizing and correlating logs, metrics, traces, exceptions
+and session replays in one place. An open source and developer-friendly
+alternative to Datadog and New Relic.
+
+<p align="center">
+  <a href="https://www.hyperdx.io/docs">Documentation</a> • <a href="https://discord.gg/FErRRKU78j">Chat on Discord</a>  • <a href="https://api.hyperdx.io/login/demo">Live Demo</a>  • Bug Reports • Contributing
+</p>
+
+- 🕵️ Correlate end to end, go from browser session replay to logs and traces in
+  just a few clicks
+- 🔥 Blazing fast performance powered by Clickhouse
+- 🔍 Intuitive full-text search and structured search syntax
+- 🤖 Automatically cluster event patterns from billions of events
+- 📈 Dashboard high cardinality events without learning a complex query language
+- 🔔 Set up alerts in just a few clicks
+- `{` Automatic JSON/structured log parsing
+- 🔭 OpenTelemetry native
+
+<br/>
+<img alt="Search logs and traces all in one place" src="./.github/images/search_splash.png" title="Search logs and traces all in one place">
+
+### Additional Screenshots
+
+<details>
+  <summary><b>📈 Dashboards</b></summary>
+  <img alt="Dashboard" src="./.github/images/dashboard.png">
+</details>
+<details>
+  <summary><b>🤖 Automatic Event Pattern Clustering</b></summary>
+  <img alt="Event Pattern Clustering" src="./.github/images/pattern3.png">
+</details>
+<details>
+  <summary><b>🖥️ Session Replay & RUM</b></summary>
+  <img alt="Event Pattern Clustering" src="./.github/images/session.png">
+</details>
+
+## Spinning Up HyperDX
+
+The HyperDX stack ingests, stores, and searches/graphs your telemetry data.
+After standing up the Docker Compose stack, you'll want to instrument your app
+to send data over to HyperDX.
+
+You can get started by deploying a complete stack via Docker Compose. After
+cloning this repository, simply start the stack with:
+
+```bash
+docker compose up
+```
+
+Afterwards, you can visit http://localhost:8080 to access the HyperDX UI.
+
+> If your server is behind a firewall, you'll need to open/forward port 8080,
+> 8000 and 4318 on your firewall for the UI, API and OTel collector
+> respectively.
+
+> We recommend at least 4GB of RAM and 2 cores for testing.
+
+**Enabling Self-instrumentation/Demo Logs**
+
+To get a quick preview of HyperDX, you can enable self-instrumentation and demo
+logs by setting the `HYPERDX_API_KEY` to your ingestion key (go to
+[http://localhost:8080/team](http://localhost:8080/team) after creating your
+account) and then restart the stack.
+
+This will redirect internal telemetry from the frontend app, API, host metrics
+and demo logs to your new HyperDX instance.
+
+ex.
+
+```sh
+HYPERDX_API_KEY=<YOUR_INGESTION_KEY> docker compose up -d
+```
+
+> If you need to use `sudo` for docker, make sure to forward the environment
+> variable with the `-E` flag:
+> `HYPERDX_API_KEY=<YOUR_KEY> sudo -E docker compose up -d`
+
+### Hosted Cloud
+
+HyperDX is also available as a hosted cloud service at
+[hyperdx.io](https://hyperdx.io). You can sign up for a free account and start
+sending data in minutes.
+
+## Instrumenting Your App
+
+To get logs, metrics, traces, session replay, etc into HyperDX, you'll need to
+instrument your app to collect and send telemetry data over to your HyperDX
+instance.
+
+We provide a set of SDKs and integration options to make it easier to get
+started with HyperDX, such as
+[Browser](https://www.hyperdx.io/docs/install/browser),
+[Node.js](https://www.hyperdx.io/docs/install/javascript), and
+[Python](https://www.hyperdx.io/docs/install/python)
+
+You can find the full list in [our docs](https://www.hyperdx.io/docs).
+
+**OpenTelemetry**
+
+Additionally, HyperDX is compatible with
+[OpenTelemetry](https://opentelemetry.io/), a vendor-neutral standard for
+instrumenting your application backed by CNCF. Supported languages/platforms
+include:
+
+- Kubernetes
+- Javascript
+- Python
+- Java
+- Go
+- Ruby
+- PHP
+- .NET
+- Elixir
+- Rust
+
+(Full list [here](https://opentelemetry.io/docs/instrumentation/))
+
+Once HyperDX is running, you can point your OpenTelemetry SDK to the
+OpenTelemetry collector spun up at `http://localhost:4318`.
+
+## Contributing
+
+We welcome all contributions! There's many ways to contribute to the project,
+including but not limited to:
+
+- Opening a PR (Contribution Guide)
+- Submitting feature requests or bugs
+- Improving our product or contribution documentation
+- Voting on open issues or contributing use cases to a feature request
+
+## Motivation
+
+Our mission is to help engineers ship reliable software. To enable that, we
+believe every engineer needs to be able to easily leverage productio telemetry
+to quickly solve burning productio issues.
+
+However, in our experience, the existing tools we've used tend to fall short in
+a few ways:
+
+1. They're expensive, and the pricing has failed to scale with TBs of telemetry
+   becoming the norm, leading to teams aggressively cutting the amount of data
+   they can collect.
+2. They're hard to use, requiring full-time SREs to set up, and domain experts
+   to use confidently.
+3. They requiring hopping from tool to tool (logs, session replay, APM,
+   exceptions, etc.) to stitch together the clues yourself.
+
+We're still early on in our journey, but are building in the open to solve these
+key issues in observability. We hope you give HyperDX a try and let us know how
+we're doing!
+
+## Open Source vs Hosted Cloud
+
+HyperDX is open core, with most of our features available here under an MIT
+license. We have a cloud-hosted version available at
+[hyperdx.io](https://hyperdx.io) with a few additional features beyond what's
+offered in the open source version.
+
+Our cloud hosted version exists so that we can build a sustainable business and
+continue building HyperDX as an open source platform. We hope to have more
+comprehensive documentation on how we balance between cloud-only and open source
+features in the future. In the meantime, we're highly aligned with Gitlab's
+[stewardship model](https://handbook.gitlab.com/handbook/company/stewardship/).
+
+## Contact
+
+- Open an Issue
+- [Discord](https://discord.gg/FErRRKU78j)
+- [Email](mailto:support@hyperdx.io)
+
+## License
+
+[MIT](/LICENSE)

docker-compose.ci.yml

@@ -0,0 +1,79 @@
+version: '3.6'
+services:
+  ch_keeper:
+    container_name: hdx-ci-ch-keeper
+    image: zookeeper:3.7
+    volumes:
+      - ./docker/clickhouse/local/zoo.cfg:/conf/zoo.cfg
+    restart: on-failure
+    networks:
+      - internal
+  ch_server:
+    container_name: hdx-ci-ch-server
+    image: clickhouse/clickhouse-server:23.5.2-alpine
+    environment:
+      # default settings
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - ./docker/clickhouse/local/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+      - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
+      - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
+    restart: on-failure
+    ports:
+      - 8123:8123 # http api
+      - 9000:9000 # native
+    networks:
+      - internal
+    depends_on:
+      - ch_keeper
+  db:
+    container_name: hdx-ci-db
+    image: mongo:5.0.14-focal
+    command: --port 29999
+    ports:
+      - 29999:29999
+    networks:
+      - internal
+  redis:
+    container_name: hdx-ci-redis
+    image: redis:7.0.11-alpine
+    ports:
+      - 6379:6379
+    networks:
+      - internal
+  api:
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+    container_name: hdx-ci-api
+    image: hyperdx/ci/api
+    ports:
+      - 9000:9000
+    environment:
+      APP_TYPE: 'api'
+      CLICKHOUSE_HOST: http://ch_server:8123
+      CLICKHOUSE_PASSWORD: api
+      CLICKHOUSE_USER: api
+      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
+      FRONTEND_URL: http://localhost:9090 # need to be localhost (CORS)
+      MONGO_URI: 'mongodb://db:29999/hyperdx-test'
+      NODE_ENV: ci
+      PORT: 9000
+      REDIS_URL: redis://redis:6379
+      SERVER_URL: http://localhost:9000
+    volumes:
+      - ./packages/api/src:/app/src
+    networks:
+      - internal
+    depends_on:
+      - ch_server
+      - db
+      - redis
+
+volumes:
+  test_mongo_data:
+
+networks:
+  internal:
+    name: 'hyperdx-ci-internal-network'

docker-compose.yml

@@ -0,0 +1,237 @@
+version: '3'
+services:
+  miner:
+    container_name: hdx-oss-dev-miner
+    build:
+      context: .
+      dockerfile: ./packages/miner/Dockerfile
+      target: dev
+    environment:
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel:4318
+      OTEL_LOG_LEVEL: ERROR
+      OTEL_SERVICE_NAME: hdx-oss-dev-miner
+    volumes:
+      - ./packages/miner/src:/app/src
+    ports:
+      - 5123:5123
+    networks:
+      - internal
+  hostmetrics:
+    container_name: hdx-oss-dev-hostmetrics
+    build:
+      context: ./docker/hostmetrics
+      target: dev
+    volumes:
+      - ./docker/hostmetrics/config.dev.yaml:/etc/otelcol-contrib/config.yaml
+    environment:
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      OTEL_SERVICE_NAME: hostmetrics
+    restart: always
+    networks:
+      - internal
+  ingestor:
+    container_name: hdx-oss-dev-ingestor
+    build:
+      context: ./docker/ingestor
+      target: dev
+    volumes:
+      - ./docker/ingestor:/app
+      - .volumes/ingestor_data:/var/lib/vector
+    ports:
+      - 8002:8002 # http-generic
+      - 8686:8686 # healthcheck
+    environment:
+      RUST_BACKTRACE: full
+      VECTOR_LOG: debug
+      VECTOR_OPENSSL_LEGACY_PROVIDER: false
+    restart: always
+    networks:
+      - internal
+  redis:
+    container_name: hdx-oss-dev-redis
+    image: redis:7.0.11-alpine
+    volumes:
+      - .volumes/redis:/data
+    ports:
+      - 6379:6379
+    networks:
+      - internal
+  db:
+    container_name: hdx-oss-dev-db
+    image: mongo:5.0.14-focal
+    volumes:
+      - .volumes/db:/data/db
+    ports:
+      - 27017:27017
+    networks:
+      - internal
+  otel:
+    container_name: hdx-oss-dev-otel
+    build:
+      context: ./docker/otel-collector
+      target: dev
+    volumes:
+      - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - '13133:13133' # health_check extension
+      - '1888:1888' # pprof extension
+      - '24225:24225' # fluentd receiver
+      - '4317:4317' # OTLP gRPC receiver
+      - '4318:4318' # OTLP http receiver
+      - '55679:55679' # zpages extension
+      - '8888:8888' # metrics extension
+      - '9411:9411' # zipkin
+    restart: always
+    networks:
+      - internal
+  aggregator:
+    container_name: hdx-oss-dev-aggregator
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+    image: hyperdx/dev/api
+    ports:
+      - 8001:8001
+    environment:
+      APP_TYPE: 'aggregator'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_PASSWORD: aggregator
+      CLICKHOUSE_USER: aggregator
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      PORT: 8001
+      REDIS_URL: redis://redis:6379
+      SERVER_URL: 'http://localhost:8000'
+    volumes:
+      - ./packages/api/src:/app/src
+    networks:
+      - internal
+    depends_on:
+      - db
+      - redis
+      - ch-server
+  task-check-alerts:
+    container_name: hdx-oss-dev-task-check-alerts
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+    image: hyperdx/dev/api
+    entrypoint: 'yarn'
+    command: 'dev:task check-alerts'
+    environment:
+      APP_TYPE: 'scheduled-task'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_LOG_LEVEL: trace
+      CLICKHOUSE_PASSWORD: worker
+      CLICKHOUSE_USER: worker
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
+      HDX_NODE_BETA_MODE: 0
+      HDX_NODE_CONSOLE_CAPTURE: 1
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_INGESTOR_ENDPOINT: 'http://ingestor:8002'
+      MINER_API_URL: 'http://miner:5123'
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-dev-task-check-alerts'
+      REDIS_URL: redis://redis:6379
+    volumes:
+      - ./packages/api/src:/app/src
+    restart: always
+    networks:
+      - internal
+    depends_on:
+      - ch-server
+      - db
+      - redis
+  api:
+    container_name: hdx-oss-dev-api
+    build:
+      context: .
+      dockerfile: ./packages/api/Dockerfile
+      target: dev
+    image: hyperdx/dev/api
+    ports:
+      - 8000:8000
+    environment:
+      APP_TYPE: 'api'
+      CLICKHOUSE_HOST: http://ch-server:8123
+      CLICKHOUSE_LOG_LEVEL: trace
+      CLICKHOUSE_PASSWORD: api
+      CLICKHOUSE_USER: api
+      EXPRESS_SESSION_SECRET: 'hyperdx is cool 👋'
+      FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS)
+      HDX_NODE_ADVANCED_NETWORK_CAPTURE: 1
+      HDX_NODE_BETA_MODE: 1
+      HDX_NODE_CONSOLE_CAPTURE: 1
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_INGESTOR_ENDPOINT: 'http://ingestor:8002'
+      MINER_API_URL: 'http://miner:5123'
+      MONGO_URI: 'mongodb://db:27017/hyperdx'
+      NODE_ENV: development
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel:4318'
+      OTEL_SERVICE_NAME: 'hdx-oss-dev-api'
+      PORT: 8000
+      REDIS_URL: redis://redis:6379
+      SERVER_URL: 'http://localhost:8000'
+    volumes:
+      - ./packages/api/src:/app/src
+    networks:
+      - internal
+    depends_on:
+      - ch-server
+      - db
+      - redis
+  app:
+    container_name: hdx-oss-dev-app
+    build:
+      context: .
+      dockerfile: ./packages/app/Dockerfile
+      target: dev
+    image: hyperdx/dev/app
+    ports:
+      - 8080:8080
+    environment:
+      NEXT_PUBLIC_API_SERVER_URL: 'http://localhost:8000' # need to be localhost (CORS)
+      NEXT_PUBLIC_HDX_API_KEY: ${HYPERDX_API_KEY}
+      NEXT_PUBLIC_HDX_COLLECTOR_URL: 'http://localhost:4318'
+      NEXT_PUBLIC_HDX_SERVICE_NAME: 'hdx-oss-dev-app'
+      NODE_ENV: development
+      PORT: 8080
+    volumes:
+      - ./packages/app/pages:/app/pages
+      - ./packages/app/public:/app/public
+      - ./packages/app/src:/app/src
+      - ./packages/app/styles:/app/styles
+      - ./packages/app/mdx.d.ts:/app/mdx.d.ts
+      - ./packages/app/next-env.d.ts:/app/next-env.d.ts
+      - ./packages/app/next.config.js:/app/next.config.js
+    networks:
+      - internal
+    depends_on:
+      - api
+  ch-server:
+    container_name: hdx-oss-dev-ch-server
+    image: clickhouse/clickhouse-server:23.7.1-alpine
+    ports:
+      - 8123:8123 # http api
+      - 9000:9000 # native
+    environment:
+      # default settings
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
+      - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
+      - .volumes/ch_data:/var/lib/clickhouse
+      - .volumes/ch_logs:/var/log/clickhouse-server
+    restart: on-failure
+    networks:
+      - internal
+networks:
+  internal:

docker/clickhouse/local/config.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <console>true</console>
+        <log remove="remove"/>
+        <errorlog remove="remove"/>
+    </logger>
+
+    <query_log>
+        <database>system</database>
+        <table>query_log</table>
+    </query_log>
+
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <interserver_http_host>ch_server</interserver_http_host>
+    <interserver_http_port>9009</interserver_http_port>
+
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>64</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+    <uncompressed_cache_size>8589934592</uncompressed_cache_size>
+    <mark_cache_size>5368709120</mark_cache_size>
+
+    <path>/var/lib/clickhouse/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+
+    <users_config>users.xml</users_config>
+    <default_profile>default</default_profile>
+    <default_database>default</default_database>
+    <timezone>UTC</timezone>
+    <mlock_executable>false</mlock_executable>
+    <!--
+        OpenTelemetry log contains OpenTelemetry trace spans.
+    -->
+    <opentelemetry_span_log>
+        <!--
+            The default table creation code is insufficient, this <engine> spec
+            is a workaround. There is no 'event_time' for this log, but two times,
+            start and finish. It is sorted by finish time, to avoid inserting
+            data too far away in the past (probably we can sometimes insert a span
+            that is seconds earlier than the last span in the table, due to a race
+            between several spans inserted in parallel). This gives the spans a
+            global order that we can use to e.g. retry insertion into some external
+            system.
+        -->
+        <engine>
+            engine MergeTree
+            partition by toYYYYMM(finish_date)
+            order by (finish_date, finish_time_us, trace_id)
+        </engine>
+        <database>system</database>
+        <table>opentelemetry_span_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </opentelemetry_span_log>
+
+    <remote_servers>
+        <hdx_cluster>
+            <shard>
+                <replica>
+                    <host>ch_server</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </hdx_cluster>
+    </remote_servers>
+
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+</clickhouse>

docker/clickhouse/local/users.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <profiles>
+        <default>
+            <max_memory_usage>10000000000</max_memory_usage>
+            <use_uncompressed_cache>0</use_uncompressed_cache>
+            <load_balancing>in_order</load_balancing>
+            <log_queries>1</log_queries>
+            <date_time_output_format>iso</date_time_output_format>
+        </default>
+    </profiles>
+
+    <users>
+        <default>
+            <password></password>
+            <profile>default</profile>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <quota>default</quota>
+        </default>
+        <api>
+            <password>api</password>
+            <profile>default</profile>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <quota>default</quota>
+        </api>
+        <aggregator>
+            <password>aggregator</password>
+            <profile>default</profile>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <quota>default</quota>
+        </aggregator>
+        <worker>
+            <password>worker</password>
+            <profile>default</profile>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <quota>default</quota>
+        </worker>
+    </users>
+
+    <quotas>
+        <default>
+            <interval>
+                <duration>3600</duration>
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+</clickhouse>

docker/hostmetrics/Dockerfile

@@ -0,0 +1,8 @@
+## base #############################################################################################
+FROM otel/opentelemetry-collector-contrib:0.83.0 AS base
+
+
+## dev #############################################################################################
+FROM base as dev
+
+COPY ./config.dev.yaml /etc/otelcol-contrib/config.yaml

docker/hostmetrics/config.dev.yaml

@@ -0,0 +1,37 @@
+receivers:
+  redis:
+    endpoint: redis:6379
+    collection_interval: 5s
+  mongodb:
+    hosts:
+      - endpoint: db:27017
+    collection_interval: 5s
+    initial_delay: 1s
+    tls:
+      insecure: true
+      insecure_skip_verify: true
+  hostmetrics:
+    collection_interval: 5s
+    scrapers:
+      cpu:
+      load:
+      memory:
+      disk:
+      filesystem:
+      network:
+exporters:
+  logging:
+    loglevel: debug
+  otlphttp:
+    endpoint: 'http://otel:4318'
+    headers:
+      authorization: ${HYPERDX_API_KEY}
+    compression: gzip
+service:
+  telemetry:
+    logs:
+      level: 'debug'
+  pipelines:
+    metrics:
+      receivers: [mongodb, redis, hostmetrics]
+      exporters: [otlphttp, logging]

docker/ingestor/Dockerfile

@@ -0,0 +1,18 @@
+## base #############################################################################################
+FROM timberio/vector:0.32.1-alpine AS base
+
+RUN mkdir -p /var/lib/vector
+VOLUME ["/var/lib/vector"]
+
+WORKDIR /app
+
+COPY ./*.toml ./
+
+
+## dev #############################################################################################
+FROM base as dev
+
+EXPOSE 8002 8686
+
+ENTRYPOINT ["vector", "-c", "http-server.core.toml", "-c", "http-server.sinks.toml", "--require-healthy", "true"]
+

docker/ingestor/http-server.core.toml

@@ -0,0 +1,753 @@
+# Set global options
+data_dir = "/var/lib/vector"
+acknowledgements.enabled = true
+
+# --------------------------------------------------------------------------------
+# ------------------------------ Sources -----------------------------------------
+# --------------------------------------------------------------------------------
+[api]
+  enabled = true
+  address = "0.0.0.0:8686"
+
+[sources.http_server]
+type = "http_server"
+address = "0.0.0.0:8002"
+headers = ["authorization", "Content-Type", "Traceparent"]
+strict_path = false
+path = ""
+query_parameters = [
+  "hdx_platform",
+  "hdx_token",
+  "sentry_key",
+  "sentry_version"
+]
+# --------------------------------------------------------------------------------
+
+
+# --------------------------------------------------------------------------------
+# ------------------------------ Middleware --------------------------------------
+# --------------------------------------------------------------------------------
+# WARNING: used for logs and spans only
+[transforms.process_headers_n_params]
+type = "remap"
+inputs = ["http_server"]
+source = '''
+  .hdx_content_type = del(."Content-Type")
+  .hdx_trace_id = split(del(.Traceparent), "-")[1] ?? null
+
+  # Hack sentry 😎
+  if is_string(.sentry_key) && length(to_string(.sentry_key) ?? "") == 32 { 
+    .hdx_content_type = "application/json"
+    .hdx_platform = "sentry"
+    .hdx_token = (slice(.sentry_key, start: 0, end: 8) + "-" + slice(.sentry_key, start: 8, end: 12) + "-" + slice(.sentry_key, start: 12, end: 16) + "-" + slice(.sentry_key, start: 16, end: 20) + "-" + slice(.sentry_key, start: 20, end: 32)) ?? null
+  }
+'''
+[transforms.unnest_jsons]
+type = "remap"
+inputs = ["process_headers_n_params"]
+source = '''
+  if !includes(["otel-metrics"], .hdx_platform) && .hdx_content_type == "application/json" {
+    structured, err = parse_json(.message)
+    if err != null {
+      log("Unable to parse JSON: " + err, level: "warn")
+    } else {
+      .message = structured
+      if is_array(.message) {
+        ., err = unnest(.message)
+        if err != null {
+          log("unnest failed: " + err, level: "error")
+        }
+      }
+    }
+  }
+'''
+[transforms.extract_token]
+type = "remap"
+inputs = ["unnest_jsons"]
+source = '''
+  if is_nullish(.hdx_token) {
+    if includes(["otel-traces"], .hdx_platform) {
+      .hdx_token = del(.message.JaegerTag.__HDX_API_KEY)
+    } else {
+      .hdx_token = split(del(.authorization), " ")[1] ?? null
+      if is_nullish(.hdx_token) {
+        .hdx_token = del(.message.__HDX_API_KEY)
+      }
+    }
+    # TODO: support metrics
+  }
+
+  # check if token is in uuid format
+  if !match(to_string(.hdx_token) ?? "", r'^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$') {
+    log("Invalid token: " + (to_string(.hdx_token) ?? ""), level: "warn")
+    .hdx_token = null
+  }
+
+  if !is_nullish(.hdx_token) {
+    .hdx_token_hash = md5(.hdx_token)
+  }
+'''
+[transforms.pre_filter]
+type = "remap"
+inputs = ["extract_token"]
+source = '''
+  hdx_message_size = strlen(encode_json(.))
+  if hdx_message_size > 6000000 {
+    token = to_string(.hdx_token) ?? ""
+    log("Message size too large: " + to_string(hdx_message_size) + " bytes, token = " + token, level: "warn")
+    # HACK: so the downstream filter can drop it
+    .message = {}
+    .hdx_token = null
+  }
+'''
+# --------------------------------------------------------------------------------
+
+
+# --------------------------------------------------------------------------------
+# ------------------------------ Logs Transform ----------------------------------
+# --------------------------------------------------------------------------------
+[transforms.filter_logs]
+type = "filter"
+inputs = ["pre_filter"]
+condition = '''
+  !includes(["otel-traces", "otel-metrics"], .hdx_platform) && !is_nullish(.hdx_token)
+'''
+
+[transforms.logs]
+type = "remap"
+inputs = ["filter_logs"]
+drop_on_abort = true
+drop_on_error = true
+reroute_dropped = true
+source = '''
+.r = del(.message)
+if .hdx_platform == "vector-internal" {
+  .ts = to_unix_timestamp(parse_timestamp(del(.timestamp), format: "%+") ?? now(), unit: "nanoseconds")
+  .b = .
+  .b.message = .r
+  .b._hdx_body = .r
+  .h = .b.host
+  .st = downcase(.b.metadata.level) ?? null
+  .sv = .hdx_platform
+  del(.b.r)
+  del(.b.hdx_platform)
+  del(.b.hdx_token)
+  .r = .b
+} else if .hdx_platform == "sentry" {
+  .b = .r
+  if (to_int(.sentry_version) ?? 0) >= 7 && is_object(.b.contexts) {
+    .ts = (to_float(.b.timestamp) ?? 0.0) * 1000000000
+    .b._hdx_body = .b.message
+    .h = .b.server_name
+    .st = downcase(.b.level) ?? "fatal"
+
+    # extract service name
+    if exists(.b.contexts.hyperdx.serviceName) {
+      .sv = .b.contexts.hyperdx.serviceName
+    } else {
+      .sv = .b.platform
+    }
+
+    # extract request metadata
+    if is_object(.b.request) {
+      .b.span.kind = "server" # only for UI
+      .b.http.method = upcase(.b.request.method) ?? ""
+      .b.http.url = .b.request.url
+      .b.http.request.header = .b.request.headers
+      del(.b.request)
+    }
+
+    # extract body message
+    if is_nullish(.b._hdx_body) {
+      if is_array(.b.exception.values) {
+        exception_type = .b.exception.values[0].type
+        exception_value = .b.exception.values[0].value
+        if is_string(.b.transaction) {
+          exception_value = .b.transaction
+        } 
+        .b._hdx_body = (exception_type + ": " + exception_value) ?? ""
+      } else {
+        .b._hdx_body = "<unlabeled event>"
+      }
+    }
+
+    # user tags
+    if is_object(.b.user) {
+      .b.userEmail = del(.b.user.email)
+      .b.userId = del(.b.user.id)
+      .b.userName = del(.b.user.username)
+    }
+
+    # promote span id and trace id
+    .b.span_id = .b.contexts.trace.span_id
+    .b.trace_id = .hdx_trace_id # use "Traceparent" header
+    if is_nullish(.b.trace_id) {
+      .b.trace_id = .b.contexts.trace.trace_id
+    }
+  }
+} else if .hdx_content_type == "application/logplex-1" {
+  tmp_raw_msg = string(.r) ?? ""
+  tmp_attrs = parse_regex(tmp_raw_msg, r'^(\d+) <(\d+)>(\d+) (\S+) (\S+) (\S+) (\S+) - (.*)', numeric_groups: true) ?? null
+  if !is_object(tmp_attrs) {
+    log("Unable to parse logplex: '" + tmp_raw_msg + "', token: " + to_string(.hdx_token) ?? "", level: "warn")
+    del(.hdx_token)
+    del(.hdx_platform)
+  } else {
+    # convert .r into an object
+    .r.message = tmp_raw_msg
+
+    tmp_priority = to_int(tmp_attrs."2") ?? null
+    if is_integer(tmp_priority) {
+      .st = to_syslog_level(mod(tmp_priority, 8) ?? 6) ?? null
+    }
+    .h = tmp_attrs."5"
+    .sv = (tmp_attrs."6" + "-" + tmp_attrs."7") ?? ""
+    tmp_body = tmp_attrs."8"
+
+    .b = {}
+    .b._hdx_body = tmp_body
+    .ts = to_unix_timestamp(parse_timestamp(del(tmp_attrs."4"), format: "%+") ?? now(), unit: "nanoseconds")
+
+    if contains(.sv, "heroku") {
+      structured = parse_key_value(tmp_body, accept_standalone_key: false) ?? null
+      if is_object(structured) {
+        .b = merge(.b, structured, deep: true) ?? .b
+        .h = .b.host
+      }
+      if exists(.b.at) {
+        .st = downcase(.b.at) ?? null
+      }
+
+      # parse integer fields
+      tmp_int_fields = [
+        "connect",
+        "service",
+        "status",
+        "bytes"
+      ]
+      for_each(tmp_int_fields) -> |_index, field| {
+        tmp_value = get(value: .b, path: [field]) ?? null
+        tmp_parsed_values = parse_regex(tmp_value, r'(\d+)', numeric_groups: true) ?? null
+        if is_object(tmp_parsed_values) {
+          tmp_parsed_value = to_int(tmp_parsed_values."0") ?? null
+          if is_integer(tmp_parsed_value) {
+            .b = set(value: .b, path: [field], data: tmp_parsed_value) ?? .b
+          }
+        }
+      }
+
+      for_each(.b) -> |key, value| {
+        if starts_with(key, "sample#") {
+          tmp_parsed_values = parse_regex(value, r'[-+]?(?:\d*\.*\d+)', numeric_groups: true) ?? null
+          if is_object(tmp_parsed_values) {
+            tmp_parsed_value = to_float(tmp_parsed_values."0") ?? null
+            if is_float(tmp_parsed_value) {
+              .b = set(value: .b, path: [key], data: tmp_parsed_value) ?? .b
+            }
+          }
+        }
+      }
+    } else {
+      parsed = false
+
+      # ruby logs
+      if !parsed {
+        structured = parse_regex(tmp_body, r'\[(.*) #(\d+)\]\s*(\S+) -- : (.*)', numeric_groups: true) ?? null
+        if is_object(structured) {
+          parsed = true
+          .b.pid = structured."2"
+          .b._hdx_body = structured."4"
+          .st = downcase(structured."3") ?? null
+        }
+      }
+
+      # JSON
+      if !parsed {
+        structured = parse_json(tmp_body) ?? null
+        if is_object(structured) {
+          parsed = true
+          .b = merge(.b, structured, deep: true) ?? .b
+          if !is_nullish(.b.message) {
+            .b._hdx_body = .b.message
+          }
+          if !is_nullish(.b.level) {
+            .st = downcase(.b.level) ?? null
+          }
+        }
+      }
+    }
+  }
+} else if contains(to_string(.hdx_content_type) ?? "", "text/plain", case_sensitive: false) {
+  .b = parse_json(.r) ?? .r
+  if is_object(.b) {
+    .r = .b
+    if .hdx_platform == "go" {
+      .b._hdx_body = .b.message
+      .st = downcase(.b.level) ?? null
+      .sv = del(.b.__hdx_sv)
+      .h = del(.b.__hdx_h)
+      .ts = to_unix_timestamp(parse_timestamp(.b.ts, format: "%+") ?? now(), unit: "nanoseconds")
+    }
+  }
+} else if .hdx_content_type == "application/json" {
+  .b = .r
+  if is_object(.b) {
+    if .hdx_platform == "nodejs" {
+      tmp_hdx = del(.b.__hdx)
+      .r = .b
+      .b._hdx_body = tmp_hdx.b
+      .h = tmp_hdx.h
+      .st = downcase(tmp_hdx.st) ?? null
+      .sv = tmp_hdx.sv
+      .ts = to_unix_timestamp(parse_timestamp(tmp_hdx.ts, format: "%+") ?? now(), unit: "nanoseconds")
+    } else if .hdx_platform == "elixir" {
+      .h = del(.b.__hdx_h)
+      .st = downcase(.b.erl_level) ?? downcase(.b.level) ?? null
+      .sv = del(.b.__hdx_sv)
+      .ts = to_unix_timestamp(parse_timestamp(.b.timestamp, format: "%+") ?? now(), unit: "nanoseconds")
+      .b._hdx_body = .b.message
+      structured = parse_json(.b.message) ?? null
+      if is_object(structured) {
+        .b = merge(.b, structured, deep: true) ?? .b
+      }
+    } else if .hdx_platform == "flyio" {
+      .h = .b.host
+      tmp_level = downcase(.b.log.level) ?? downcase(.b.log.severity) ?? "info"
+      if tmp_level != "info" {
+        .st = tmp_level
+      }
+      .sv = .b.fly.app.name
+      .ts = to_unix_timestamp(parse_timestamp(.b.timestamp, format: "%+") ?? now(), unit: "nanoseconds")
+      .b._hdx_body = .b.message
+      structured = parse_json(.b.message) ?? null
+      if is_object(structured) {
+        .b = merge(.b, structured, deep: true) ?? .b
+      }
+      # TODO: maybe move this to post_logs
+      if !is_nullish(.b.message) {
+        .b._hdx_body = .b.message
+      }
+    } else if .hdx_platform == "vercel" {
+      .h = .b.host
+      .st = downcase(.b.level) ?? downcase(.b.severity) ?? null
+      .sv = .b.projectName
+      .ts = (to_int(.b.timestamp) ?? 0) * 1000000
+
+      # default to message
+      .b._hdx_body = .b.message
+
+      # build up custom message (apache http log format)
+      if exists(.b.proxy) {
+        # set status code
+        tmp_status = to_int(.b.proxy.statusCode) ?? 200
+        if tmp_status >= 500 {
+          .st = "error"
+        }
+ 
+        .b._hdx_body = .b.proxy.clientIp + " - " + "\"" + (join([
+          .b.proxy.method,
+          .b.proxy.path,
+        ], separator: " ") ?? "") + "\"" + " " + to_string(.b.proxy.statusCode) ?? ""
+      }
+
+      # attach trace id
+      .b.trace_id = .b.requestId
+
+      # extract more props
+      if .b.source == "lambda" {
+        .b.Duration = to_float(parse_regex(.b.message, r'Duration: (?P<d>.*?) ms').d ?? null) ?? null
+        .b.BilledDuration = to_int(parse_regex(.b.message, r'Billed Duration: (?P<d>.*?) ms').d ?? null) ?? null
+        .b.InitDuration = to_float(parse_regex(.b.message, r'Init Duration: (?P<d>.*?) ms').d ?? null) ?? null
+        .b.MemorySize = to_int(parse_regex(.b.message, r'Memory Size: (?P<d>.*?) MB').d ?? null) ?? null
+        .b.MaxMemoryUsed = to_int(parse_regex(.b.message, r'Max Memory Used: (?P<d>.*?) MB').d ?? null) ?? null
+
+        tmp_splits = split(.b.message, "\n") ?? []
+        tmp_logs = []
+
+        for_each(tmp_splits) -> |_index, value| {
+          if !is_nullish(value) {
+            tmp_cur_log = {}
+            tmp_cur_log._hdx_body = value
+
+            tmp_msg_splits = split(value, "\t")
+            for_each(tmp_msg_splits) -> |__index, tmp_msg_split| {
+              if starts_with(tmp_msg_split, "{") && ends_with(tmp_msg_split, "}") {
+                _structured = parse_json(tmp_msg_split) ?? null
+                if is_object(_structured) {
+                  tmp_cur_log = merge(tmp_cur_log, _structured, deep: false) ?? tmp_cur_log
+                }
+              }
+            }
+
+            tmp_logs = push(tmp_logs, tmp_cur_log)
+          }
+        }
+
+        if length(tmp_logs) > 0 {
+          # will be split into multiple logs
+          .__hdx_logs = tmp_logs
+        }
+      }
+    } else if .hdx_platform == "aws-lambda" {
+      tmp_timestamp = to_int(.b."@timestamp") ?? 0
+      .b._hdx_body = .b.message
+      .ts = to_unix_timestamp(from_unix_timestamp(tmp_timestamp, unit: "milliseconds") ?? now(), unit: "nanoseconds")
+      .sv = del(.b.type)
+      if is_nullish(.sv) {
+        .sv = .b.logGroup
+      }
+      structured = parse_json(.b.message) ?? null
+      if is_object(structured) {
+        .b = merge(.b, structured, deep: true) ?? .b
+      }
+    } else if .hdx_platform == "aws-sns" {
+      .st = "ok"
+      .b._hdx_body = .b.Message
+      .ts = to_unix_timestamp(parse_timestamp(.b.Timestamp, format: "%+") ?? now(), unit: "nanoseconds")
+      structured = parse_json(.b.Message) ?? null
+      if is_object(structured) {
+        .b = merge(.b, structured, deep: true) ?? .b
+      }
+    } else if .hdx_platform == "otel-logs" {
+      del(.r."@timestamp")
+      tmp_timestamp = del(.b."@timestamp")
+      
+      .r.timestamp = tmp_timestamp
+      .b.timestamp = tmp_timestamp
+
+      .h = .b.host
+      .st = downcase(.b.level) ?? null
+      .sv = .b."service.name"
+      .ts = to_unix_timestamp(from_unix_timestamp(.b.timestamp, unit: "milliseconds") ?? now(), unit: "nanoseconds")
+      .b._hdx_body = .b.message
+      structured = parse_json(.b.message) ?? null
+      if is_object(structured) {
+        .b = merge(.b, structured, deep: true) ?? .b
+      }
+
+      if exists(.b."rr-web.event") {
+        .hdx_platform = "rrweb"
+        temp_msg = .b.message
+        temp_msg_json = parse_json(temp_msg) ?? null
+        temp_rum_session_id = .b."rum.sessionId"
+        temp_rum_script_instance = .b."rum.scriptInstance"
+        temp_rrweb = {
+          "event": .b."rr-web.event",
+          "offset": .b."rr-web.offset",
+          "chunk": .b."rr-web.chunk",
+          "total-chunks": .b."rr-web.total-chunks"
+        }
+        .b = {}
+        .b._hdx_body = temp_msg
+        if is_object(temp_msg_json) {
+          .b.type = temp_msg_json.type
+        }
+        .b."rum.sessionId" = temp_rum_session_id
+        .b."rum.scriptInstance" = temp_rum_script_instance
+        .b."rr-web" = temp_rrweb
+      }
+    }
+  } else {
+    del(.b)
+  }
+} 
+del(.source_type)
+del(.timestamp)
+del(.authorization)
+del(.hdx_content_type)
+del(.hdx_trace_id)
+del(.sentry_key)
+del(.sentry_version)
+'''
+
+[transforms.post_logs_unnest]
+type = "remap"
+inputs = ["logs"]
+drop_on_abort = true
+drop_on_error = true
+reroute_dropped = true
+source = '''
+if is_array(.__hdx_logs) {
+  ., err = unnest(.__hdx_logs)
+  if err != null {
+    log("unnest failed: " + err, level: "error")
+  }
+}
+'''
+
+[transforms.post_logs]
+type = "remap"
+inputs = ["post_logs_unnest"]
+drop_on_abort = true
+drop_on_error = true
+reroute_dropped = true
+source = '''
+# extract shared fields
+if is_object(.b) {
+  .s_id = string(del(.b.span_id)) ?? string(del(.b.spanID)) ?? null
+  .t_id = string(del(.b.trace_id)) ?? string(del(.b.traceID)) ?? null
+  .tso = to_unix_timestamp(now(), unit: "nanoseconds")
+
+  if is_nullish(.st) {
+    .st = downcase(.b.level) ?? downcase(.b.severity) ?? downcase(.b.LEVEL) ?? downcase(.b.SEVERITY) ?? null
+  }
+
+  # merge vercel logs
+  if is_object(.__hdx_logs) {
+    tmp_b_size = strlen(encode_json(.b))
+    tmp_hdx_logs_size = strlen(encode_json(.__hdx_logs))
+    tmp_total_size = tmp_b_size + tmp_hdx_logs_size
+
+    # Max expanded log size 16MB
+    if tmp_total_size > 16000000 {
+      log("__hdx_logs + body size too large: " + to_string(tmp_total_size) + " bytes, token = " + to_string(.hdx_token) ?? "", level: "warn")
+      del(.__hdx_logs)
+    } else {
+      .b = merge(.b, del(.__hdx_logs), deep: false) ?? .b
+    }
+
+    .b.message = .b._hdx_body
+    if is_object(.r) {
+      .r.message = .b._hdx_body
+    }
+  }
+
+  # add _hdx_body to raw logs (make it searchable)
+  if is_object(.r) {
+    if !is_nullish(.b._hdx_body) && !contains(encode_json(.r), to_string(.b._hdx_body) ?? "", case_sensitive: false) {
+      .r._hdx_body = .b._hdx_body
+    }
+  }
+}
+
+# validate timestamp and set it to observed timestamp if it is invalid
+if exists(.ts) {
+  a_day_from_now = to_unix_timestamp(now(), unit: "nanoseconds") + 86400000000000
+  nighty_days_ago = to_unix_timestamp(now(), unit: "nanoseconds") - 90 * 86400000000000
+  if (.ts > a_day_from_now ?? false) || (.ts < nighty_days_ago ?? false) {
+    .ts = .tso
+  }
+}
+
+# infer log level for raw logs
+if is_nullish(.st) {
+  header = ""
+  if is_object(.r) && !is_nullish(.b._hdx_body) {
+    header = slice(to_string(.b._hdx_body) ?? "", start: 0, end: 160) ?? ""
+  } else {
+    header = slice(to_string(.r) ?? "", start: 0, end: 160) ?? ""
+  }
+  if contains(header, "info", case_sensitive: false) {
+    .st = "info"
+  } else if contains(header, "warn", case_sensitive: false) {
+    .st = "warn"
+  } else if contains(header, "error", case_sensitive: false) {
+    .st = "error"
+  } else if contains(header, "debug", case_sensitive: false) {
+    .st = "debug"
+  } else if contains(header, "trace", case_sensitive: false) {
+    .st = "trace"
+  } else if contains(header, "fatal", case_sensitive: false) {
+    .st = "fatal"
+  } else if contains(header, "notice", case_sensitive: false) {
+    .st = "notice"
+  } else if contains(header, "crit", case_sensitive: false) {
+    .st = "critical"
+  } else if contains(header, "emerg", case_sensitive: false) {
+    .st = "emergency"
+  } else if contains(header, "alert", case_sensitive: false) {
+    .st = "alert"
+  } else {
+    .st = "info" # mike said that on 2023-07-23
+  }
+}
+
+# TODO: compute sn ?
+.sn = 0
+'''
+# --------------------------------------------------------------------------------
+
+
+# --------------------------------------------------------------------------------
+# ----------------------------- Spans Transform ----------------------------------
+# --------------------------------------------------------------------------------
+[transforms.filter_spans]
+type = "filter"
+inputs = ["pre_filter"]
+condition = '''
+  includes(["otel-traces"], .hdx_platform) && !is_nullish(.hdx_token)
+'''
+
+[transforms.spans]
+type = "remap"
+inputs = ["filter_spans"]
+drop_on_abort = true
+drop_on_error = true
+reroute_dropped = true
+source = '''
+.r = del(.message)
+if is_object(.r) {
+  tmp_JaegerTag = map_keys(object(.r.JaegerTag) ?? {}, recursive: true) -> |key| { replace(key,"@",".") }
+  tmp_process = map_keys(object(.r.process) ?? {}, recursive: true) -> |key| { replace(key,"@",".") } 
+  tmp_timestamp = del(.r."@timestamp")
+  .r.timestamp = tmp_timestamp
+  .r.JaegerTag = tmp_JaegerTag
+  .r.process = tmp_process
+  .s_n = .r.operationName
+  .s_id = .r.spanID
+  .t_id = .r.traceID
+  .p_id = null
+  ref = .r.references[0]
+  if (ref != null && ref.refType == "CHILD_OF") {
+    .p_id = ref.spanID
+  }
+  .b = tmp_JaegerTag
+  .b.process = tmp_process
+  .b.__events = .r.logs
+
+  if (.b."span.kind" == "server") {
+    if (exists(.b."http.status_code") && exists(.b."http.method") && exists(.b."http.route")) {
+      .b._hdx_body = join([
+        to_string(.b."http.status_code") ?? "",
+        .b."http.method",
+        .b."http.route",
+      ], separator: " ") ?? .s_n
+    }
+  } else if (.b."span.kind" == "client") {
+    if (exists(.b."http.status_code") && exists(.b."http.method") && exists(.b."http.url")) {
+      .b._hdx_body = join([
+        to_string(.b."http.status_code") ?? "",
+        .b."http.method",
+        .b."http.url",
+      ], separator: " ") ?? .s_n
+    }
+  } else if (.b."span.kind" == "internal") {
+    if .b.component == "console" {
+      .b._hdx_body = .b.message
+      .st = .b.level
+    }
+  }
+
+  if ((to_int(.b.error) ?? 0) == 1) {
+    .st = "error"
+    if !is_nullish(.b."otel.status_description") {
+      .b._hdx_body = .b."otel.status_description"
+    } else if !is_nullish(.b."error.message") {
+      .b._hdx_body = .b."error.message"
+    }
+  } else if is_array(.b.__events) {
+    for_each(array(.b.__events) ?? []) -> |_index, value| {
+      if is_object(value) {
+        if (value.fields[0].key == "event" && value.fields[0].value == "exception") {
+          .st = "error"
+        }
+      }
+    }
+  }
+
+  # set default values
+  if is_nullish(.st) {
+    .st = "ok"
+  }
+  if is_nullish(.b._hdx_body) {
+    .b._hdx_body = .s_n
+  }
+
+  # RN instrumentation
+  if exists(.b."process.serviceName") {
+    .sv = .b."process.serviceName"
+  } else {
+    .sv = .r.process.serviceName
+  }
+
+  .ts = to_unix_timestamp(from_unix_timestamp(.r.startTimeMillis, unit: "milliseconds") ?? now(), unit: "nanoseconds")
+  .et = .ts + .r.duration * 1000 ?? 0
+  .tso = to_unix_timestamp(now(), unit: "nanoseconds")
+
+  # TODO: move this to post_spans
+  # add _hdx_body to raw log
+  if !is_nullish(.b._hdx_body) && !contains(encode_json(.r), to_string(.b._hdx_body) ?? "", case_sensitive: false) {
+    .r._hdx_body = .b._hdx_body
+  }
+
+  del(.source_type)
+  del(.timestamp)
+  del(.authorization)
+  del(.hdx_content_type)
+}
+'''
+# --------------------------------------------------------------------------------
+
+
+# --------------------------------------------------------------------------------
+# ---------------------------- Metrics Transform ---------------------------------
+# --------------------------------------------------------------------------------
+[transforms.filter_metrics]
+type = "filter"
+inputs = ["http_server"]
+condition = '''
+  includes(["otel-metrics"], .hdx_platform)
+'''
+
+[transforms.pre_metrics]
+type = "remap"
+inputs = ["filter_metrics"]
+source = '''
+  del(.path)
+  del(.source_type)
+  del(.timestamp)
+  del(.authorization)
+  del(.hdx_content_type)
+  tmp_msg = del(.message)
+  .message = split(tmp_msg, "}}") ?? []
+  . = unnest(.message)
+'''
+
+[transforms.metrics]
+type = "remap"
+inputs = ["pre_metrics"]
+drop_on_abort = true
+drop_on_error = true
+reroute_dropped = true
+source = '''
+  tmp = (del(.message) + "}}") ?? null
+  structured, err = parse_json(tmp)
+  if err == null && structured.event == "metric" {
+    # TODO: do this at extract_token
+    .hdx_token = del(structured.fields.__HDX_API_KEY)
+    .dt = structured.fields.metric_type
+    filtered_keys = ["metric_type"]
+    for_each(object(structured.fields) ?? {})-> |key, value| {
+      if is_integer(value) || is_float(value) {
+        filtered_keys = push(filtered_keys, key)
+        .n = replace(key, "metric_name:", "")
+        .v = value
+      }
+    }
+    .tso = to_unix_timestamp(now(), unit: "nanoseconds")
+    .ts = to_int(structured.time * 1000000000 ?? null) ?? .tso
+    .b = filter(object(structured.fields) ?? {})-> |key, value| {
+      !includes(filtered_keys, key)
+    }
+    .b.host = structured.host
+  }
+'''
+# --------------------------------------------------------------------------------
+
+
+# --------------------------------------------------------------------------------
+# --------------------------------- Debug ----------------------------------------
+# --------------------------------------------------------------------------------
+[transforms.debug_dropped]
+type = "remap"
+inputs = [
+  "logs.dropped",
+  "metrics.dropped",
+  "post_logs.dropped",
+  "post_logs_unnest.dropped",
+  "spans.dropped"
+]
+source = '''
+  log(., level: "error")
+'''
+# --------------------------------------------------------------------------------

docker/ingestor/http-server.sinks.toml

@@ -0,0 +1,21 @@
+[sinks.dev_hdx_aggregator]
+type = "http"
+uri = "http://aggregator:8001"
+inputs = ["spans", "post_logs"]
+compression = "gzip"
+encoding.codec = "json"
+batch.max_bytes = 10485760 # 10MB, required for rrweb payloads 
+batch.max_events = 100
+batch.timeout_secs = 1
+
+
+[sinks.dev_hdx_metrics_aggregator]
+type = "http"
+uri = "http://aggregator:8001?telemetry=metric"
+inputs = ["metrics"]
+compression = "gzip"
+encoding.codec = "json"
+batch.max_bytes = 100000
+batch.max_events = 100
+batch.timeout_secs = 1
+# --------------------------------------------------------------------------------

docker/ingestor/run_linting.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+directory="./docker/ingestor"
+
+vector validate --no-environment $directory/*.toml

docker/otel-collector/Dockerfile

@@ -0,0 +1,10 @@
+## base #############################################################################################
+FROM otel/opentelemetry-collector-contrib:0.83.0 AS base
+
+
+## dev #############################################################################################
+FROM base as dev
+
+COPY ./config.yaml /etc/otelcol-contrib/config.yaml
+
+EXPOSE 1888 4317 4318 55679 13133

docker/otel-collector/config.yaml

@@ -0,0 +1,86 @@
+receivers:
+  # Data sources: traces
+  zipkin:
+    endpoint: '0.0.0.0:9411'
+  # Data sources: logs
+  fluentforward:
+    endpoint: '0.0.0.0:24225'
+  # Data sources: traces, metrics, logs
+  otlp:
+    protocols:
+      grpc:
+        include_metadata: true
+        endpoint: '0.0.0.0:4317'
+      http:
+        cors:
+          allowed_origins: ['*']
+          allowed_headers: ['*']
+        include_metadata: true
+        endpoint: '0.0.0.0:4318'
+processors:
+  resourcedetection:
+    detectors:
+      - env
+      - system
+      - docker
+    timeout: 5s
+    override: false
+  attributes/attachHdxKey:
+    actions:
+      - key: __HDX_API_KEY
+        from_context: authorization
+        action: upsert
+  batch:
+  memory_limiter:
+    # 80% of maximum memory up to 2G
+    limit_mib: 1500
+    # 25% of limit up to 2G
+    spike_limit_mib: 512
+    check_interval: 5s
+exporters:
+  logging:
+    loglevel: debug
+  logzio/traces:
+    account_token: 'X' # required but we don't use it
+    endpoint: 'http://ingestor:8002?hdx_platform=otel-traces'
+  logzio/logs:
+    account_token: 'X' # required but we don't use it
+    endpoint: 'http://ingestor:8002?hdx_platform=otel-logs'
+  # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter
+  splunk_hec:
+    token: 'X' # required but we don't use it
+    # endpoint: 'http://ingestor:8002?hdx_platform=otel-metrics'
+    endpoint: 'http://ingestor:8002?hdx_platform=otel-metrics'
+    # HTTP timeout when sending data. Defaults to 10s.
+    timeout: 10s
+    max_content_length_logs: 0
+extensions:
+  health_check:
+    endpoint: :13133
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+  memory_ballast:
+    # Memory Ballast size should be max 1/3 to 1/2 of memory.
+    size_mib: 683
+service:
+  telemetry:
+    metrics:
+      address: ':8888'
+    logs:
+      level: 'debug'
+  extensions: [health_check, zpages, memory_ballast]
+  pipelines:
+    traces:
+      receivers: [otlp, zipkin]
+      processors: [attributes/attachHdxKey, memory_limiter, batch]
+      exporters: [logzio/traces, logging]
+    metrics:
+      receivers: [otlp]
+      processors: [attributes/attachHdxKey, memory_limiter, batch]
+      exporters: [splunk_hec, logging]
+    logs:
+      receivers: [otlp, fluentforward]
+      processors: [attributes/attachHdxKey, memory_limiter, batch]
+      exporters: [logzio/logs, logging]

nx.json

@@ -0,0 +1,22 @@
+{
+  "affected": {
+    "defaultBase": "main"
+  },
+  "workspaceLayout": {
+    "appsDir": "packages",
+    "libsDir": "packages"
+  },
+  "targetDefaults": {
+    "build": {
+      "dependsOn": ["^build"]
+    }
+  },
+  "tasksRunnerOptions": {
+    "default": {
+      "runner": "nx/tasks-runners/default",
+      "options": {
+        "cacheableOperations": ["build", "lint", "test"]
+      }
+    }
+  }
+}

package.json

@@ -0,0 +1,25 @@
+{
+  "private": true,
+  "version": "0.0.0",
+  "license": "MIT",
+  "workspaces": [
+    "packages/*"
+  ],
+  "devDependencies": {
+    "@nx/workspace": "16.8.1",
+    "husky": "^8.0.3",
+    "lint-staged": "^13.1.2",
+    "nx": "16.8.1",
+    "prettier": "2.8.4"
+  },
+  "scripts": {
+    "prepare": "husky install",
+    "dev:lint": "./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint",
+    "ci:lint": "./docker/ingestor/run_linting.sh && yarn workspaces run ci:lint",
+    "dev:int": "docker compose -p int -f ./docker-compose.ci.yml run --rm api dev:int",
+    "ci:int": "docker compose -p int -f ./docker-compose.ci.yml run --rm api ci:int"
+  },
+  "lint-staged": {
+    "**/*": "prettier --write --ignore-unknown"
+  }
+}

packages/api/.Dockerignore

@@ -0,0 +1,2 @@
+**/node_modules
+**/build

packages/api/.eslintignore

@@ -0,0 +1,3 @@
+keys
+node_modules
+archive

packages/api/.eslintrc.js

@@ -0,0 +1,24 @@
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint', 'prettier'],
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: ['./tsconfig.json'],
+  },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/eslint-recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:prettier/recommended',
+  ],
+  rules: {
+    '@typescript-eslint/ban-ts-comment': 'warn',
+    '@typescript-eslint/no-empty-interface': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+    '@typescript-eslint/no-floating-promises': 'error',
+    '@typescript-eslint/no-namespace': 'warn',
+    '@typescript-eslint/no-unused-vars': 'warn',
+    'prettier/prettier': 'error',
+  },
+};

packages/api/Dockerfile

@@ -0,0 +1,18 @@
+## base #############################################################################################
+FROM node:18.15.0-alpine AS base
+
+WORKDIR /app
+
+COPY .yarn ./.yarn
+COPY .yarnrc yarn.lock ./packages/api/jest.config.js ./packages/api/tsconfig.json ./packages/api/package.json ./
+RUN yarn install --frozen-lockfile && yarn cache clean
+
+
+## dev #############################################################################################
+
+FROM base AS dev
+
+EXPOSE 8000
+
+ENTRYPOINT ["yarn"]
+CMD ["dev"]

packages/api/jest.config.js

@@ -0,0 +1,9 @@
+/** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  verbose: true,
+  rootDir: './src',
+  testMatch: ['**/__tests__/*.test.ts?(x)'],
+  testTimeout: 30000,
+};

packages/api/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "@hyperdx/api",
+  "version": "1.0.0",
+  "license": "MIT",
+  "private": true,
+  "engines": {
+    "node": ">=18.12.0"
+  },
+  "dependencies": {
+    "@clickhouse/client": "^0.1.1",
+    "@hyperdx/lucene": "^3.1.1",
+    "@hyperdx/node-opentelemetry": "^0.2.2",
+    "@slack/webhook": "^6.1.0",
+    "compression": "^1.7.4",
+    "connect-mongo": "^4.6.0",
+    "cors": "^2.8.5",
+    "date-fns": "^2.28.0",
+    "date-fns-tz": "^2.0.0",
+    "express": "^4.17.3",
+    "express-rate-limit": "^6.7.1",
+    "express-session": "^1.17.3",
+    "express-winston": "^4.2.0",
+    "extract-domain": "^2.4.1",
+    "isemail": "^3.2.0",
+    "jsonwebtoken": "^9.0.0",
+    "lodash": "^4.17.21",
+    "minimist": "^1.2.7",
+    "mongoose": "^6.12.0",
+    "ms": "^2.1.3",
+    "node-schedule": "^2.1.1",
+    "object-hash": "^3.0.0",
+    "on-headers": "^1.0.2",
+    "passport": "^0.5.3",
+    "passport-local": "^1.0.0",
+    "passport-local-mongoose": "^6.1.0",
+    "pluralize": "^8.0.0",
+    "rate-limit-redis": "^3.0.2",
+    "redis": "^4.6.8",
+    "semver": "^7.5.2",
+    "serialize-error": "^8.1.0",
+    "sqlstring": "^2.3.3",
+    "uuid": "^8.3.2",
+    "winston": "^3.10.0",
+    "zod": "^3.22.2",
+    "zod-express-middleware": "^1.4.0"
+  },
+  "devDependencies": {
+    "@slack/types": "^2.8.0",
+    "@types/airbnb__node-memwatch": "^2.0.0",
+    "@types/compression": "^1.7.3",
+    "@types/cors": "^2.8.14",
+    "@types/express": "^4.17.13",
+    "@types/express-session": "^1.17.7",
+    "@types/extract-domain": "^2.3.1",
+    "@types/hyperdx__lucene": "npm:@types/lucene",
+    "@types/jest": "^28.1.1",
+    "@types/lodash": "^4.14.198",
+    "@types/minimist": "^1.2.2",
+    "@types/ms": "^0.7.31",
+    "@types/object-hash": "^2.2.1",
+    "@types/passport-http-bearer": "^1.0.37",
+    "@types/passport-local": "^1.0.34",
+    "@types/pluralize": "^0.0.29",
+    "@types/semver": "^7.3.12",
+    "@types/sqlstring": "^2.3.0",
+    "@types/supertest": "^2.0.12",
+    "@types/uuid": "^8.3.4",
+    "@typescript-eslint/eslint-plugin": "^5.46.1",
+    "@typescript-eslint/parser": "^5.46.1",
+    "eslint": "^8.48.0",
+    "eslint-config-prettier": "^8.5.0",
+    "eslint-plugin-prettier": "^4.2.1",
+    "jest": "^28.1.1",
+    "nodemon": "^2.0.20",
+    "rimraf": "^4.4.1",
+    "supertest": "^6.3.1",
+    "ts-jest": "^28.0.5",
+    "ts-node": "^10.8.1",
+    "typescript": "^4.9.5"
+  },
+  "scripts": {
+    "start": "node ./build/index.js",
+    "dev": "nodemon --signal SIGTERM -e ts,json --exec 'ts-node' --transpile-only -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/index.ts",
+    "dev:task": "ts-node -r '@hyperdx/node-opentelemetry/build/src/tracing' ./src/tasks/index.ts",
+    "build": "rimraf ./build && tsc",
+    "lint": "eslint . --ext .ts",
+    "ci:lint": "yarn lint && yarn tsc --noEmit",
+    "ci:int": "jest --runInBand --ci --forceExit --coverage",
+    "dev:int": "jest --watchAll --runInBand --detectOpenHandles"
+  }
+}

packages/api/src/aggregator-app.ts

@@ -0,0 +1,50 @@
+import compression from 'compression';
+import express from 'express';
+
+import * as clickhouse from './clickhouse';
+import logger, { expressLogger } from './utils/logger';
+import routers from './routers/aggregator';
+import { appErrorHandler } from './middleware/error';
+import { mongooseConnection } from './models';
+
+import type { Request, Response, NextFunction } from 'express';
+
+const app: express.Application = express();
+
+const healthCheckMiddleware = async (
+  req: Request,
+  res: Response,
+  next: NextFunction,
+) => {
+  if (mongooseConnection.readyState !== 1) {
+    logger.error('MongoDB is down!');
+    return res.status(500).send('MongoDB is down!');
+  }
+
+  try {
+    await clickhouse.healthCheck();
+  } catch (e) {
+    logger.error('Clickhouse is down!');
+    return res.status(500).send('Clickhouse is down!');
+  }
+  next();
+};
+
+app.disable('x-powered-by');
+app.use(compression());
+app.use(express.json({ limit: '64mb' }));
+app.use(express.text({ limit: '64mb' }));
+app.use(express.urlencoded({ extended: false, limit: '64mb' }));
+
+app.use(expressLogger);
+
+// ---------------------------------------------------------
+// -------------------- Routers ----------------------------
+// ---------------------------------------------------------
+app.use('/', healthCheckMiddleware, routers.rootRouter);
+// ---------------------------------------------------------
+
+// error handling
+app.use(appErrorHandler);
+
+export default app;

packages/api/src/api-app.ts

@@ -0,0 +1,75 @@
+import MongoStore from 'connect-mongo';
+import compression from 'compression';
+import express from 'express';
+import session from 'express-session';
+import onHeaders from 'on-headers';
+
+import * as config from './config';
+import defaultCors from './middleware/cors';
+import passport from './utils/passport';
+import routers from './routers/api';
+import { appErrorHandler } from './middleware/error';
+import { expressLogger } from './utils/logger';
+
+const app: express.Application = express();
+
+const sess: session.SessionOptions & { cookie: session.CookieOptions } = {
+  resave: false,
+  saveUninitialized: false,
+  secret: config.EXPRESS_SESSION_SECRET,
+  cookie: {
+    secure: false,
+    maxAge: 1000 * 60 * 60 * 24 * 30, // 30 days
+  },
+  rolling: true,
+  store: new MongoStore({ mongoUrl: config.MONGO_URI }),
+};
+
+if (config.IS_PROD) {
+  app.set('trust proxy', 1); // Super important or cookies don't get set in prod
+  sess.cookie.secure = true;
+  sess.cookie.domain = config.COOKIE_DOMAIN;
+}
+
+app.disable('x-powered-by');
+app.use(compression());
+app.use(express.json({ limit: '32mb' }));
+app.use(express.text({ limit: '32mb' }));
+app.use(express.urlencoded({ extended: false, limit: '32mb' }));
+app.use(session(sess));
+
+app.use(passport.initialize());
+app.use(passport.session());
+
+app.use(expressLogger);
+// Allows timing data from frontend package
+// see: https://github.com/expressjs/cors/issues/102
+app.use(function (req, res, next) {
+  onHeaders(res, function () {
+    const allowOrigin = res.getHeader('Access-Control-Allow-Origin');
+    if (allowOrigin) {
+      res.setHeader('Timing-Allow-Origin', allowOrigin);
+    }
+  });
+  next();
+});
+app.use(defaultCors);
+
+// ---------------------------------------------------------------------
+// ----------------------- Internal Routers ----------------------------
+// ---------------------------------------------------------------------
+app.use('/', routers.rootRouter);
+app.use('/alerts', routers.alertsRouter);
+app.use('/dashboards', routers.dashboardRouter);
+app.use('/log-views', routers.logViewsRouter);
+app.use('/logs', routers.logsRouter);
+app.use('/metrics', routers.metricsRouter);
+app.use('/sessions', routers.sessionsRouter);
+app.use('/team', routers.teamRouter);
+app.use('/webhooks', routers.webhooksRouter);
+// ---------------------------------------------------------------------
+
+// error handling
+app.use(appErrorHandler);
+
+export default app;

packages/api/src/clickhouse/__tests__/clickhouse.test.ts

@@ -0,0 +1,45 @@
+import * as clickhouse from '..';
+
+describe('clickhouse', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('clientInsertWithRetries (success)', async () => {
+    jest
+      .spyOn(clickhouse.client, 'insert')
+      .mockRejectedValueOnce(new Error('first error'))
+      .mockRejectedValueOnce(new Error('second error'))
+      .mockResolvedValueOnce(null as any);
+
+    await clickhouse.clientInsertWithRetries({
+      table: 'testTable',
+      values: [{ test: 'test' }],
+      retries: 3,
+      timeout: 100,
+    });
+
+    expect(clickhouse.client.insert).toHaveBeenCalledTimes(3);
+  });
+
+  it('clientInsertWithRetries (fail)', async () => {
+    jest
+      .spyOn(clickhouse.client, 'insert')
+      .mockRejectedValueOnce(new Error('first error'))
+      .mockRejectedValueOnce(new Error('second error'));
+
+    try {
+      await clickhouse.clientInsertWithRetries({
+        table: 'testTable',
+        values: [{ test: 'test' }],
+        retries: 2,
+        timeout: 100,
+      });
+    } catch (error: any) {
+      expect(error.message).toBe('second error');
+    }
+
+    expect(clickhouse.client.insert).toHaveBeenCalledTimes(2);
+    expect.assertions(2);
+  });
+});

packages/api/src/clickhouse/__tests__/searchQueryParser.test.ts

@@ -0,0 +1,563 @@
+import SqlString from 'sqlstring';
+
+import { LogsPropertyTypeMappingsModel } from '../propertyTypeMappingsModel';
+import {
+  SearchQueryBuilder,
+  genWhereSQL,
+  msToBigIntNs,
+  parse,
+} from '../searchQueryParser';
+
+describe('searchQueryParser', () => {
+  describe('helpers', () => {
+    it('msToBigIntNs', () => {
+      expect(msToBigIntNs(0)).toBe(BigInt(0));
+      expect(msToBigIntNs(1000)).toBe(BigInt(1000000000));
+    });
+  });
+
+  // for implicit field
+  function implicitLike(column: string, term: string) {
+    return `(lower(${column}) LIKE lower('${term}'))`;
+  }
+
+  function implicitLikeSubstring(column: string, term: string) {
+    return `(lower(${column}) LIKE lower('%${term}%'))`;
+  }
+
+  function like(column: string, term: string) {
+    return `(${column} ILIKE '${term}')`;
+  }
+  function likeSubstring(column: string, term: string) {
+    return `(${column} ILIKE '%${term}%')`;
+  }
+  function nlike(column: string, term: string) {
+    return `(${column} NOT ILIKE '%${term}%')`;
+  }
+  function hasToken(column: string, term: string, noParen = false) {
+    return `${noParen ? '' : '('}hasTokenCaseInsensitive(${column}, '${term}')${
+      noParen ? '' : ')'
+    }`;
+  }
+  function notHasToken(column: string, term: string) {
+    return `(NOT hasTokenCaseInsensitive(${column}, '${term}'))`;
+  }
+  function eq(column: string, term: string, isExpression = false) {
+    return `(${column} = ${isExpression ? '' : "'"}${term}${
+      isExpression ? '' : "'"
+    })`;
+  }
+  function neq(column: string, term: string, isExpression = false) {
+    return `(${column} != ${isExpression ? '' : "'"}${term}${
+      isExpression ? '' : "'"
+    })`;
+  }
+  function range(column: string, min: string, max: string) {
+    return `(${column} BETWEEN ${min} AND ${max})`;
+  }
+  function nrange(column: string, min: string, max: string) {
+    return `(${column} NOT BETWEEN ${min} AND ${max})`;
+  }
+  function buildSearchColumnName(
+    type: 'string' | 'number' | 'bool',
+    name: string,
+  ) {
+    return SqlString.format(`_${type}_attributes[?]`, [name]);
+  }
+
+  const SOURCE_COL = '_source';
+
+  let propertyTypesMappingsModel: LogsPropertyTypeMappingsModel;
+
+  beforeEach(() => {
+    propertyTypesMappingsModel = new LogsPropertyTypeMappingsModel(
+      1,
+      'fake team id',
+      () => Promise.resolve({}),
+    );
+    jest
+      .spyOn(propertyTypesMappingsModel, 'get')
+      .mockReturnValue(undefined as any);
+    jest.spyOn(propertyTypesMappingsModel, 'refresh').mockResolvedValue();
+  });
+
+  it('SearchQueryBuilder', async () => {
+    jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+    const builder = new SearchQueryBuilder(
+      'level:info OR level:warn',
+      propertyTypesMappingsModel,
+    );
+    builder.timestampInBetween(
+      new Date('2019-01-01').getTime(),
+      new Date('2019-01-02').getTime(),
+    );
+    const query = await builder.build();
+    expect(query).toBe(
+      "(_timestamp_sort_key >= 1546300800000000000 AND _timestamp_sort_key < 1546387200000000000) AND ((severity_text ILIKE '%info%') OR (severity_text ILIKE '%warn%'))",
+    );
+  });
+
+  describe('bare terms', () => {
+    it('parses simple bare terms', async () => {
+      const ast = parse('foo');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(hasToken(SOURCE_COL, 'foo'));
+    });
+
+    it('parses multiple bare terms', async () => {
+      const ast = parse('foo bar baz999');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        `${hasToken(SOURCE_COL, 'foo')} AND ${hasToken(
+          SOURCE_COL,
+          'bar',
+        )} AND ${hasToken(SOURCE_COL, 'baz999')}`,
+      );
+    });
+
+    it('parses quoted bare terms', async () => {
+      const ast = parse('"foo" "bar" baz999');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        `${hasToken(SOURCE_COL, 'foo')} AND ${hasToken(
+          SOURCE_COL,
+          'bar',
+        )} AND ${hasToken(SOURCE_COL, 'baz999')}`,
+      );
+    });
+
+    it('parses quoted multi-terms', async () => {
+      const ast = parse('"foo bar"');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        `(${hasToken(SOURCE_COL, 'foo', true)} AND ${hasToken(
+          SOURCE_COL,
+          'bar',
+          true,
+        )} AND ${implicitLikeSubstring(SOURCE_COL, 'foo bar')})`,
+      );
+    });
+
+    it('parses bare terms with symbols', async () => {
+      const ast = parse('scott!');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        `(${hasToken(SOURCE_COL, 'scott', true)} AND ${implicitLikeSubstring(
+          SOURCE_COL,
+          'scott!',
+        )})`,
+      );
+    });
+
+    it('parses quoted bare terms with symbols', async () => {
+      const ast = parse('"scott["');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        `(${hasToken(SOURCE_COL, 'scott', true)} AND ${implicitLikeSubstring(
+          SOURCE_COL,
+          'scott[',
+        )})`,
+      );
+    });
+
+    // TODO: Figure out symbol handling here as well...
+    it.skip('does not do comparison operators on quoted bare terms', async () => {
+      const ast = parse('"<foo>"');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        `(${hasToken(SOURCE_COL, '<foo>')}} AND ${likeSubstring(
+          SOURCE_COL,
+          '<foo>',
+        )})`,
+      );
+    });
+
+    describe('parentheses', () => {
+      it('parses parenthesized bare terms', async () => {
+        const ast = parse('foo (bar baz)');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(
+          `${hasToken(SOURCE_COL, 'foo')} AND (${hasToken(
+            SOURCE_COL,
+            'bar',
+          )} AND ${hasToken(SOURCE_COL, 'baz')})`,
+        );
+      });
+
+      it('parses parenthesized negated bare terms', async () => {
+        const ast = parse('foo (-bar baz)');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(
+          `${hasToken(SOURCE_COL, 'foo')} AND (${notHasToken(
+            SOURCE_COL,
+            'bar',
+          )} AND ${hasToken(SOURCE_COL, 'baz')})`,
+        );
+      });
+    });
+
+    describe('negation', () => {
+      it('negates bare terms', async () => {
+        const ast = parse('-bar');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(`${notHasToken(SOURCE_COL, 'bar')}`);
+      });
+
+      it('negates quoted bare terms', async () => {
+        const ast = parse('-"bar baz"');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(
+          `(NOT (${hasToken(SOURCE_COL, 'bar', true)} AND ${hasToken(
+            SOURCE_COL,
+            'baz',
+            true,
+          )} AND ${implicitLikeSubstring(SOURCE_COL, 'bar baz')}))`,
+        );
+      });
+
+      it('matches negated and non-negated bare terms', async () => {
+        const ast = parse('foo -bar baz -qux');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(
+          `${hasToken(SOURCE_COL, 'foo')} AND ${notHasToken(
+            SOURCE_COL,
+            'bar',
+          )} AND ${hasToken(SOURCE_COL, 'baz')} AND ${notHasToken(
+            SOURCE_COL,
+            'qux',
+          )}`,
+        );
+      });
+    });
+
+    describe('wildcards', () => {
+      it('allows wildcard prefix and postfix', async () => {
+        const ast = parse('*foo*');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(`${implicitLike(SOURCE_COL, '%foo%')}`);
+      });
+
+      it('does not parse * in the middle of terms', async () => {
+        const ast = parse('ff*oo*');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(`${implicitLike(SOURCE_COL, 'ff*oo%')}`);
+      });
+
+      // TODO: Handle this
+      it.skip('does not parse * in quoted terms', async () => {
+        const ast = parse('"*foobar baz"');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(`${hasToken(SOURCE_COL, '*foo*bar baz')}`);
+      });
+    });
+  });
+
+  describe('properties', () => {
+    it('parses string property values', async () => {
+      const ast = parse('foo:bar');
+      jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(likeSubstring("_string_attributes['foo']", 'bar'));
+    });
+
+    it('parses bool property values', async () => {
+      const ast = parse('foo:1');
+      jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('bool');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        // "(_bool_attributes['foo'] = 1)",
+        `${eq(buildSearchColumnName('bool', 'foo'), '1', true)}`,
+      );
+    });
+
+    it('parses numeric property values', async () => {
+      const ast = parse('foo:123');
+      jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('number');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        `${eq(
+          buildSearchColumnName('number', 'foo'),
+          "CAST('123', 'Float64')",
+          true,
+        )}`,
+      );
+    });
+
+    it('parses hex property values', async () => {
+      const ast = parse('foo:0fa1b0ba');
+      jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(likeSubstring("_string_attributes['foo']", '0fa1b0ba'));
+    });
+
+    it('parses quoted property values', async () => {
+      const ast = parse('foo:"blah:foo http://website"');
+      jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(eq("_string_attributes['foo']", 'blah:foo http://website'));
+    });
+
+    it('parses bare terms combined with property values', async () => {
+      const ast = parse('bar foo:0f');
+      jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(
+        `${hasToken(SOURCE_COL, 'bar')} AND ${likeSubstring(
+          "_string_attributes['foo']",
+          '0f',
+        )}`,
+      );
+    });
+
+    it('parses ranges of values', async () => {
+      const ast = parse('foo:[400 TO 599]');
+      jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(`${range("_string_attributes['foo']", '400', '599')}`);
+    });
+
+    it('parses numeric properties', async () => {
+      const ast = parse('5:info');
+      jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(`${likeSubstring("_string_attributes['5']", 'info')}`);
+    });
+
+    it('translates custom column mapping', async () => {
+      const ast = parse('level:info');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual(`${likeSubstring('severity_text', 'info')}`);
+    });
+
+    it('handle non-existent property', async () => {
+      const ast = parse('foo:bar');
+      expect(
+        await genWhereSQL(
+          ast,
+          propertyTypesMappingsModel,
+          'TEAM_ID_UNIT_TESTS',
+        ),
+      ).toEqual('(1 = 0)');
+    });
+
+    describe('negation', () => {
+      it('negates property values', async () => {
+        const ast = parse('-foo:bar');
+        jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(`${nlike(buildSearchColumnName('string', 'foo'), 'bar')}`);
+      });
+
+      it('supports negated negative property string values', async () => {
+        const ast = parse('-foo:-bar');
+        jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(`${nlike(buildSearchColumnName('string', 'foo'), '-bar')}`);
+      });
+
+      it('supports negated negative property number values', async () => {
+        const ast = parse('-foo:-5');
+        jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('number');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(
+          `${neq(
+            buildSearchColumnName('number', 'foo'),
+            "CAST('-5', 'Float64')",
+            true,
+          )}`,
+        );
+      });
+
+      it('supports negating numeric properties', async () => {
+        const ast = parse('-5:info');
+        jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(`${nlike(buildSearchColumnName('string', '5'), 'info')}`);
+      });
+
+      it('supports negating numeric properties with negative values', async () => {
+        const ast = parse('-5:-150');
+        jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('number');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(
+          `${neq(
+            buildSearchColumnName('number', '5'),
+            "CAST('-150', 'Float64')",
+            true,
+          )}`,
+        );
+      });
+
+      it('negates ranges of values', async () => {
+        const ast = parse('-5:[-100 TO -500]');
+        jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(
+          `${nrange(buildSearchColumnName('string', '5'), '-100', '-500')}`,
+        );
+      });
+
+      it('negates quoted searches', async () => {
+        const ast = parse('-foo:"bar"');
+        jest.spyOn(propertyTypesMappingsModel, 'get').mockReturnValue('string');
+        expect(
+          await genWhereSQL(
+            ast,
+            propertyTypesMappingsModel,
+            'TEAM_ID_UNIT_TESTS',
+          ),
+        ).toEqual(`${neq(buildSearchColumnName('string', 'foo'), 'bar')}`);
+      });
+    });
+  });
+});

packages/api/src/clickhouse/propertyTypeMappingsModel.ts

@@ -0,0 +1,195 @@
+import ms from 'ms';
+import { serializeError } from 'serialize-error';
+
+import logger from '../utils/logger';
+import redisClient from '../utils/redis';
+import { IS_DEV } from '../config';
+
+import type { ResponseJSON } from '@clickhouse/client';
+
+const stringifyMap = (map: Map<any, any>) => {
+  return JSON.stringify(Array.from(map.entries()));
+};
+
+const parseStringifiedMap = (stringifiedMap: string): Map<string, any> => {
+  return new Map(JSON.parse(stringifiedMap));
+};
+
+export abstract class PropertyTypeMappingsModel {
+  abstract getCacheKey(): string;
+
+  abstract ttl(): number;
+
+  protected readonly tableVersion: number | undefined;
+
+  protected readonly teamId: string;
+
+  private readonly fetchPropertyTypeMappings: (
+    tableVersion: number | undefined,
+    teamId: string,
+  ) => Promise<ResponseJSON<Record<string, any[]>>>;
+
+  // hold the mapping state
+  public currentPropertyTypeMappings: Map<string, 'string' | 'number' | 'bool'>;
+
+  constructor(
+    tableVersion: number | undefined,
+    teamId: string,
+    fetchPropertyTypeMappings: (
+      tableVersion: number | undefined,
+      teamId: string,
+    ) => Promise<any>,
+  ) {
+    this.tableVersion = tableVersion;
+    this.teamId = teamId;
+    this.fetchPropertyTypeMappings = fetchPropertyTypeMappings;
+    this.currentPropertyTypeMappings = new Map();
+  }
+
+  private bundlePropertyTypeMappings(
+    raw: ResponseJSON<Record<string, any[]>> | null,
+  ) {
+    const mapping = new Map<string, 'string' | 'number' | 'bool'>();
+    if (raw?.rows === 1) {
+      const data = raw.data[0];
+      data.strings?.map((property: string) => mapping.set(property, 'string'));
+      data.numbers?.map((property: string) => mapping.set(property, 'number'));
+      data.bools?.map((property: string) => mapping.set(property, 'bool'));
+    }
+    return mapping;
+  }
+
+  // decide if the cache is still valid
+  private async needsRefresh() {
+    return true;
+  }
+
+  // only used internally
+  async _refresh({ incrementalUpdate }: { incrementalUpdate: boolean }) {
+    logger.info({
+      message: 'propertyTypeMappingsModel _refresh start',
+      teamId: this.teamId,
+    });
+    try {
+      const mappings = await this.fetchPropertyTypeMappings(
+        this.tableVersion,
+        this.teamId,
+      );
+      const oldMappings = this.currentPropertyTypeMappings;
+      const newMappings = this.bundlePropertyTypeMappings(mappings);
+      this.currentPropertyTypeMappings = incrementalUpdate
+        ? new Map([...oldMappings, ...newMappings]) // WARNING: newMappings will overwrite oldMappings
+        : newMappings;
+
+      if (incrementalUpdate) {
+        // if incrementalUpdate = true, we only update the value but keep the TTL
+        await redisClient.set(
+          this.getCacheKey(),
+          stringifyMap(this.currentPropertyTypeMappings),
+          {
+            KEEPTTL: true,
+          },
+        );
+      } else {
+        await redisClient.set(
+          this.getCacheKey(),
+          stringifyMap(this.currentPropertyTypeMappings),
+          {
+            PX: this.ttl(),
+          },
+        );
+      }
+
+      logger.info({
+        incrementalUpdate,
+        message: 'propertyTypeMappingsModel _refresh success',
+        teamId: this.teamId,
+      });
+    } catch (err) {
+      logger.error({
+        error: serializeError(err),
+        incrementalUpdate,
+        message: 'propertyTypeMappingsModel _refresh error',
+        teamId: this.teamId,
+      });
+    }
+  }
+
+  _debug() {
+    console.log([...this.currentPropertyTypeMappings.entries()]);
+  }
+
+  async init() {
+    const cachedMappings: any = await redisClient.get(this.getCacheKey());
+    if (cachedMappings) {
+      logger.info({
+        message: 'propertyTypeMappingsModel init: cache hit',
+        teamId: this.teamId,
+      });
+      this.currentPropertyTypeMappings = parseStringifiedMap(cachedMappings);
+    } else {
+      logger.info({
+        message: 'propertyTypeMappingsModel init: cache miss',
+        teamId: this.teamId,
+      });
+      // cache miss
+      await this._refresh({ incrementalUpdate: false });
+    }
+  }
+
+  // used by the APIs
+  async refresh() {
+    logger.info({
+      message: 'propertyTypeMappingsModel refresh: start',
+      teamId: this.teamId,
+    });
+    if (await this.needsRefresh()) {
+      logger.info({
+        message: 'propertyTypeMappingsModel refresh: cache miss',
+        teamId: this.teamId,
+      });
+      await this._refresh({ incrementalUpdate: true });
+    } else {
+      logger.info({
+        message: 'propertyTypeMappingsModel refresh: cache hit',
+        teamId: this.teamId,
+      });
+    }
+  }
+
+  get(property: string) {
+    return this.currentPropertyTypeMappings.get(property);
+  }
+
+  size() {
+    return this.currentPropertyTypeMappings.size;
+  }
+
+  remainingTTL() {
+    return redisClient.pTTL(this.getCacheKey());
+  }
+
+  async isAboutToExpire() {
+    return (await this.remainingTTL()) < ms('2h');
+  }
+}
+
+export class LogsPropertyTypeMappingsModel extends PropertyTypeMappingsModel {
+  getCacheKey() {
+    return `logs_property_type_mappings:${this.teamId}`;
+  }
+
+  ttl() {
+    return IS_DEV ? ms('5s') : ms('1d');
+  }
+}
+
+export class MetricsPropertyTypeMappingsModel extends PropertyTypeMappingsModel {
+  getCacheKey() {
+    return `metrics_property_type_mappings:${this.teamId}`;
+  }
+
+  ttl() {
+    return IS_DEV ? ms('5s') : ms('1d');
+  }
+}

packages/api/src/clickhouse/searchQueryParser.ts

@@ -0,0 +1,627 @@
+import SqlString from 'sqlstring';
+import lucene from '@hyperdx/lucene';
+import { serializeError } from 'serialize-error';
+
+import { LogPlatform, LogType } from '../utils/logParser';
+import { PropertyTypeMappingsModel } from './propertyTypeMappingsModel';
+
+function encodeSpecialTokens(query: string): string {
+  return query
+    .replace('http://', 'http_COLON_//')
+    .replace('https://', 'https_COLON_//')
+    .replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1')
+    .replace(/\\:/g, 'HDX_COLON');
+}
+function decodeSpecialTokens(query: string): string {
+  return query
+    .replace('http_COLON_//', 'http://')
+    .replace('https_COLON_//', 'https://')
+    .replace(/localhost_COLON_(\d{1,5})/, 'localhost:$1')
+    .replace(/HDX_COLON/g, ':');
+}
+
+export function parse(query: string): lucene.AST {
+  return lucene.parse(encodeSpecialTokens(query));
+}
+
+const IMPLICIT_FIELD = '<implicit>';
+
+export const msToBigIntNs = (ms: number) => BigInt(ms * 1000000);
+export const isLikelyTokenTerm = (term: string) => {
+  return term.length >= 16;
+};
+
+const customColumnMap: { [level: string]: string } = {
+  [IMPLICIT_FIELD]: '_source',
+  body: '_hdx_body',
+  duration: '_duration',
+  end_timestamp: 'end_timestamp',
+  host: '_host',
+  hyperdx_event_size: '_hyperdx_event_size',
+  hyperdx_platform: '_platform',
+  level: 'severity_text',
+  parent_span_id: 'parent_span_id',
+  rum_session_id: '_rum_session_id',
+  service: '_service',
+  span_id: 'span_id',
+  span_name: 'span_name',
+  timestamp: 'timestamp',
+  trace_id: 'trace_id',
+  userEmail: '_user_email',
+  userId: '_user_id',
+  userName: '_user_name',
+};
+export const customColumnMapType: {
+  [property: string]: 'string' | 'number' | 'bool';
+} = {
+  [IMPLICIT_FIELD]: 'string',
+  body: 'string',
+  duration: 'number',
+  host: 'string',
+  hyperdx_event_size: 'number',
+  hyperdx_platform: 'string',
+  level: 'string',
+  parent_span_id: 'string',
+  rum_session_id: 'string',
+  service: 'string',
+  span_id: 'string',
+  span_name: 'string',
+  trace_id: 'string',
+  userEmail: 'string',
+  userId: 'string',
+  userName: 'string',
+};
+
+export const isCustomColumn = (name: string) => customColumnMap[name] != null;
+
+// used by rrweb table
+export const buildSearchColumnName_OLD = (
+  type: 'string' | 'number' | 'bool',
+  name: string,
+) => {
+  if (customColumnMap[name] != null) {
+    return customColumnMap[name];
+  }
+
+  return type != null && name != null
+    ? `"${type}.values"[indexOf("${type}.names", ${SqlString.escape(name)})]`
+    : null;
+};
+
+export const buildSearchColumnName = (
+  type: 'string' | 'number' | 'bool' | undefined | null,
+  name: string,
+) => {
+  if (customColumnMap[name] != null) {
+    return customColumnMap[name];
+  }
+
+  if (name != null && type != null) {
+    return SqlString.format(`_${type}_attributes[?]`, [name]);
+  }
+
+  return null;
+};
+
+interface Serializer {
+  eq(field: string, term: string, isNegatedField: boolean): Promise<string>;
+  isNotNull(field: string, isNegatedField: boolean): Promise<string>;
+  gte(field: string, term: string): Promise<string>;
+  lte(field: string, term: string): Promise<string>;
+  lt(field: string, term: string): Promise<string>;
+  gt(field: string, term: string): Promise<string>;
+  fieldSearch(
+    field: string,
+    term: string,
+    isNegatedField: boolean,
+    prefixWildcard: boolean,
+    suffixWildcard: boolean,
+  ): Promise<string>;
+  range(
+    field: string,
+    start: string,
+    end: string,
+    isNegatedField: boolean,
+  ): Promise<string>;
+}
+
+type SQLSerializerOptions = {
+  useTokenization: boolean;
+};
+export class SQLSerializer implements Serializer {
+  private NOT_FOUND_QUERY = '(1 = 0)';
+
+  private alreadyRefrehPropertyTypeMapModel = false;
+
+  propertyTypeMapModel: PropertyTypeMappingsModel;
+
+  options: SQLSerializerOptions | undefined;
+
+  constructor(
+    propertyTypeMappingsModel: PropertyTypeMappingsModel,
+    opts?: SQLSerializerOptions,
+  ) {
+    this.propertyTypeMapModel = propertyTypeMappingsModel;
+    this.options = opts ?? {
+      useTokenization: false,
+    };
+  }
+
+  private getCustomFieldOnly(field: string) {
+    return {
+      column: customColumnMap[field],
+      propertyType: customColumnMapType[field],
+      found: customColumnMap[field] != null, // propertyType can be null
+    };
+  }
+
+  // In the future this may trigger network calls against a property mapping cache
+  async getColumnForField(field: string) {
+    const customField = this.getCustomFieldOnly(field);
+    if (customField.found) {
+      return customField;
+    }
+
+    let propertyType = this.propertyTypeMapModel.get(field);
+    // TODO: Deal with ambiguous fields
+    let column: string = field;
+    // refresh cache if property not found
+    if (propertyType == null && !this.alreadyRefrehPropertyTypeMapModel) {
+      this.alreadyRefrehPropertyTypeMapModel = true;
+      // TODO: what if the property type doesn't exist?
+      // we need to setup a cap on how many times we refresh the cache
+      await this.propertyTypeMapModel.refresh();
+      propertyType = this.propertyTypeMapModel.get(field);
+    }
+
+    if (propertyType != null) {
+      column = buildSearchColumnName(propertyType, field);
+    }
+
+    return {
+      column,
+      propertyType,
+      found: column != null && propertyType != null,
+    };
+  }
+
+  async eq(field: string, term: string, isNegatedField: boolean) {
+    const { column, found, propertyType } = await this.getColumnForField(field);
+    if (!found) {
+      return this.NOT_FOUND_QUERY;
+    }
+    if (propertyType === 'bool') {
+      // numeric and boolean fields must be equality matched
+      return SqlString.format(`(${column} ${isNegatedField ? '!' : ''}= ?)`, [
+        parseInt(term),
+      ]);
+    } else if (propertyType === 'number') {
+      return SqlString.format(
+        `(${column} ${isNegatedField ? '!' : ''}= CAST(?, 'Float64'))`,
+        [term],
+      );
+    }
+    return SqlString.format(`(${column} ${isNegatedField ? '!' : ''}= ?)`, [
+      term,
+    ]);
+  }
+
+  async isNotNull(field: string, isNegatedField: boolean) {
+    const customField = this.getCustomFieldOnly(field);
+    if (customField.found) {
+      if (field === 'duration') {
+        // Duration will be negative if there is no end_timestamp
+        return `_duration ${isNegatedField ? '<' : '>='} 0`;
+      }
+      if (customField.propertyType === 'string') {
+        // Internal string fields are not nullable as long as they are not empty, they're likely not null
+        return `notEmpty(${customField.column}) ${
+          isNegatedField ? '!' : ''
+        }= 1`;
+      } else {
+        // We'll just try to check for nulls...
+        return `${customField.column} IS ${isNegatedField ? '' : 'NOT '}NULL`;
+      }
+    }
+
+    const { propertyType, found } = await this.getColumnForField(field);
+    if (!found) {
+      return this.NOT_FOUND_QUERY;
+    }
+    return SqlString.format(`mapContains(_${propertyType}_attributes, ?) = ?`, [
+      field,
+      isNegatedField ? 0 : 1,
+    ]);
+  }
+
+  async gte(field: string, term: string) {
+    const { column, found } = await this.getColumnForField(field);
+    if (!found) {
+      return this.NOT_FOUND_QUERY;
+    }
+    return SqlString.format(`(${column} >= ?)`, [term]);
+  }
+
+  async lte(field: string, term: string) {
+    const { column, found } = await this.getColumnForField(field);
+    if (!found) {
+      return this.NOT_FOUND_QUERY;
+    }
+    return SqlString.format(`(${column} <= ?)`, [term]);
+  }
+
+  async lt(field: string, term: string) {
+    const { column, found } = await this.getColumnForField(field);
+    if (!found) {
+      return this.NOT_FOUND_QUERY;
+    }
+    return SqlString.format(`(${column} < ?)`, [term]);
+  }
+
+  async gt(field: string, term: string) {
+    const { column, found } = await this.getColumnForField(field);
+    if (!found) {
+      return this.NOT_FOUND_QUERY;
+    }
+    return SqlString.format(`(${column} > ?)`, [term]);
+  }
+
+  // TODO: Not sure if SQL really needs this or if it'll coerce itself
+  private attemptToParseNumber(term: string): string | number {
+    const number = Number.parseFloat(term);
+    if (Number.isNaN(number)) {
+      return term;
+    }
+    return number;
+  }
+
+  // Ref: https://clickhouse.com/codebrowser/ClickHouse/src/Functions/HasTokenImpl.h.html#_ZN2DB12HasTokenImpl16isTokenSeparatorEDu
+  // Split by anything that's ascii 0-128, that's not a letter or a number
+  private tokenizeTerm(term: string): string[] {
+    return term.split(/[ -/:-@[-`{-~\t\n\r]+/).filter(t => t.length > 0);
+  }
+
+  private termHasSeperators(term: string): boolean {
+    return term.match(/[ -/:-@[-`{-~\t\n\r]+/) != null;
+  }
+
+  async fieldSearch(
+    field: string,
+    term: string,
+    isNegatedField: boolean,
+    prefixWildcard: boolean,
+    suffixWildcard: boolean,
+  ) {
+    const isImplicitField = field === IMPLICIT_FIELD;
+    const { column, propertyType, found } = await this.getColumnForField(field);
+    if (!found) {
+      return this.NOT_FOUND_QUERY;
+    }
+    // If it's a string field, we will always try to match with ilike
+
+    if (propertyType === 'bool') {
+      // numeric and boolean fields must be equality matched
+      return SqlString.format(`(${column} ${isNegatedField ? '!' : ''}= ?)`, [
+        parseInt(term),
+      ]);
+    } else if (propertyType === 'number') {
+      return SqlString.format(
+        `(${column} ${isNegatedField ? '!' : ''}= CAST(?, 'Float64'))`,
+        [term],
+      );
+    }
+
+    if (isImplicitField && this.options?.useTokenization) {
+      // For the _source column, we'll try to do whole word searches by default
+      // to utilize the token bloom filter unless a prefix/sufix wildcard is specified
+      if (prefixWildcard || suffixWildcard) {
+        return SqlString.format(
+          `(lower(${column}) ${isNegatedField ? 'NOT ' : ''}LIKE lower(?))`,
+          [`${prefixWildcard ? '%' : ''}${term}${suffixWildcard ? '%' : ''}`],
+        );
+      } else {
+        // We can't search multiple tokens with `hasToken`, so we need to split up the term into tokens
+        const hasSeperators = this.termHasSeperators(term);
+        if (hasSeperators) {
+          const tokens = this.tokenizeTerm(term);
+          return `(${isNegatedField ? 'NOT (' : ''}${[
+            ...tokens.map(token =>
+              SqlString.format(`hasTokenCaseInsensitive(${column}, ?)`, [
+                token,
+              ]),
+            ),
+            // If there are symbols in the term, we'll try to match the whole term as well (ex. Scott!)
+            SqlString.format(`(lower(${column}) LIKE lower(?))`, [`%${term}%`]),
+          ].join(' AND ')}${isNegatedField ? ')' : ''})`;
+        } else {
+          return SqlString.format(
+            `(${
+              isNegatedField ? 'NOT ' : ''
+            }hasTokenCaseInsensitive(${column}, ?))`,
+            [term],
+          );
+        }
+      }
+    } else {
+      const shoudUseTokenBf = isImplicitField && isLikelyTokenTerm(term);
+      return SqlString.format(
+        `(${column} ${isNegatedField ? 'NOT ' : ''}? ?)`,
+        [SqlString.raw(shoudUseTokenBf ? 'LIKE' : 'ILIKE'), `%${term}%`],
+      );
+    }
+  }
+
+  async range(
+    field: string,
+    start: string,
+    end: string,
+    isNegatedField: boolean,
+  ) {
+    const { column, found } = await this.getColumnForField(field);
+    if (!found) {
+      return this.NOT_FOUND_QUERY;
+    }
+    return SqlString.format(
+      `(${column} ${isNegatedField ? 'NOT ' : ''}BETWEEN ? AND ?)`,
+      [this.attemptToParseNumber(start), this.attemptToParseNumber(end)],
+    );
+  }
+}
+
+async function nodeTerm(
+  node: lucene.Node,
+  serializer: Serializer,
+): Promise<string> {
+  const field = node.field[0] === '-' ? node.field.slice(1) : node.field;
+  let isNegatedField = node.field[0] === '-';
+  const isImplicitField = node.field === IMPLICIT_FIELD;
+
+  // NodeTerm
+  if ((node as lucene.NodeTerm).term != null) {
+    const nodeTerm = node as lucene.NodeTerm;
+    let term = decodeSpecialTokens(nodeTerm.term);
+    // We should only negate the search for negated bare terms (ex. '-5')
+    // This meeans the field is implicit and the prefix is -
+    if (isImplicitField && nodeTerm.prefix === '-') {
+      isNegatedField = true;
+    }
+    // Otherwise, if we have a negated term for a field (ex. 'level:-5')
+    // we should not negate the search, and search for -5
+    if (!isImplicitField && nodeTerm.prefix === '-') {
+      term = nodeTerm.prefix + decodeSpecialTokens(nodeTerm.term);
+    }
+
+    // TODO: Decide if this is good behavior
+    // If the term is quoted, we should search for the exact term in a property (ex. foo:"bar")
+    // Implicit field searches should still use substring matching (ex. "foo bar")
+    if (nodeTerm.quoted && !isImplicitField) {
+      return serializer.eq(field, term, isNegatedField);
+    }
+
+    if (!nodeTerm.quoted && term === '*') {
+      return serializer.isNotNull(field, isNegatedField);
+    }
+
+    if (!nodeTerm.quoted && term.substring(0, 2) === '>=') {
+      if (isNegatedField) {
+        return serializer.lt(field, term.slice(2));
+      }
+      return serializer.gte(field, term.slice(2));
+    }
+    if (!nodeTerm.quoted && term.substring(0, 2) === '<=') {
+      if (isNegatedField) {
+        return serializer.gt(field, term.slice(2));
+      }
+      return serializer.lte(field, term.slice(2));
+    }
+    if (!nodeTerm.quoted && term[0] === '>') {
+      if (isNegatedField) {
+        return serializer.lte(field, term.slice(1));
+      }
+      return serializer.gt(field, term.slice(1));
+    }
+    if (!nodeTerm.quoted && term[0] === '<') {
+      if (isNegatedField) {
+        return serializer.gte(field, term.slice(1));
+      }
+      return serializer.lt(field, term.slice(1));
+    }
+
+    let prefixWildcard = false;
+    let suffixWildcard = false;
+    if (!nodeTerm.quoted && term[0] === '*') {
+      prefixWildcard = true;
+      term = term.slice(1);
+    }
+    if (!nodeTerm.quoted && term[term.length - 1] === '*') {
+      suffixWildcard = true;
+      term = term.slice(0, -1);
+    }
+
+    return serializer.fieldSearch(
+      field,
+      term,
+      isNegatedField,
+      prefixWildcard,
+      suffixWildcard,
+    );
+
+    // TODO: Handle regex, similarity, boost, prefix
+  }
+  // NodeRangedTerm
+  if ((node as lucene.NodeRangedTerm).inclusive != null) {
+    const rangedTerm = node as lucene.NodeRangedTerm;
+    return serializer.range(
+      field,
+      rangedTerm.term_min,
+      rangedTerm.term_max,
+      isNegatedField,
+    );
+  }
+
+  throw new Error(`Unexpected Node type. ${node}`);
+}
+
+async function serialize(
+  ast: lucene.AST | lucene.Node,
+  serializer: Serializer,
+): Promise<string> {
+  // Node Scenarios:
+  // 1. NodeTerm: Single term ex. "foo:bar"
+  // 2. NodeRangedTerm: Two terms ex. "foo:[bar TO qux]"
+  if ((ast as lucene.NodeTerm).term != null) {
+    return await nodeTerm(ast as lucene.NodeTerm, serializer);
+  }
+  if ((ast as lucene.NodeRangedTerm).inclusive != null) {
+    return await nodeTerm(ast as lucene.NodeTerm, serializer);
+  }
+
+  // AST Scenarios:
+  // 1. BinaryAST: Two terms ex. "foo:bar AND baz:qux"
+  // 2. LeftOnlyAST: Single term ex. "foo:bar"
+  if ((ast as lucene.BinaryAST).right != null) {
+    const binaryAST = ast as lucene.BinaryAST;
+    const operator =
+      binaryAST.operator === IMPLICIT_FIELD ? 'AND' : binaryAST.operator;
+    const parenthesized = binaryAST.parenthesized;
+    return `${parenthesized ? '(' : ''}${await serialize(
+      binaryAST.left,
+      serializer,
+    )} ${operator} ${await serialize(binaryAST.right, serializer)}${
+      parenthesized ? ')' : ''
+    }`;
+  }
+
+  if ((ast as lucene.LeftOnlyAST).left != null) {
+    const leftOnlyAST = ast as lucene.LeftOnlyAST;
+    const parenthesized = leftOnlyAST.parenthesized;
+    // start is used when ex. "NOT foo:bar"
+    return `${parenthesized ? '(' : ''}${
+      leftOnlyAST.start != undefined ? `${leftOnlyAST.start} ` : ''
+    }${await serialize(leftOnlyAST.left, serializer)}${
+      parenthesized ? ')' : ''
+    }`;
+  }
+
+  // Blank AST, means no text was parsed
+  return '';
+}
+
+export async function genWhereSQL(
+  ast: lucene.AST,
+  propertyTypeMappingsModel: PropertyTypeMappingsModel,
+  teamId?: string,
+): Promise<string> {
+  const serializer = new SQLSerializer(propertyTypeMappingsModel, {
+    useTokenization: true,
+  });
+  return await serialize(ast, serializer);
+}
+
+export class SearchQueryBuilder {
+  private readonly searchQ: string;
+
+  private readonly conditions: string[];
+
+  private readonly propertyTypeMappingsModel: PropertyTypeMappingsModel;
+
+  teamId?: string;
+
+  constructor(
+    searchQ: string,
+    propertyTypeMappingsModel: PropertyTypeMappingsModel,
+  ) {
+    this.conditions = [];
+    this.searchQ = searchQ;
+    this.propertyTypeMappingsModel = propertyTypeMappingsModel;
+  }
+
+  private async genSearchQuery() {
+    if (!this.searchQ) {
+      return '';
+    }
+
+    let querySql = this.searchQ
+      .split(/\s+/)
+      .map(queryToken =>
+        SqlString.format(`lower(_source) LIKE lower(?)`, [`%${queryToken}%`]),
+      )
+      .join(' AND ');
+
+    try {
+      const parsedQ = parse(this.searchQ);
+
+      if (parsedQ) {
+        querySql = await genWhereSQL(
+          parsedQ,
+          this.propertyTypeMappingsModel,
+          this.teamId,
+        );
+      }
+    } catch (e) {
+      console.warn({
+        error: serializeError(e),
+        message: 'Parse failure',
+        query: this.searchQ,
+      });
+    }
+
+    return querySql;
+  }
+
+  and(condition: string) {
+    if (condition && condition.trim()) {
+      this.conditions.push(`(${condition})`);
+    }
+    return this;
+  }
+
+  removeInternals() {
+    this.and(SqlString.format('notEquals(_platform, ?)', [LogPlatform.Rrweb]));
+    return this;
+  }
+
+  filterLogsAndSpans() {
+    this.and(
+      SqlString.format('type = ? OR type = ?', [LogType.Log, LogType.Span]),
+    );
+    return this;
+  }
+
+  // startTime and endTime are unix in ms
+  timestampInBetween(startTime: number, endTime: number) {
+    this.and(
+      `_timestamp_sort_key >= ${msToBigIntNs(
+        startTime,
+      )} AND _timestamp_sort_key < ${msToBigIntNs(endTime)}`,
+    );
+    return this;
+  }
+
+  async build() {
+    const searchQuery = await this.genSearchQuery();
+    if (this.searchQ) {
+      this.and(searchQuery);
+    }
+    return this.conditions.join(' AND ');
+  }
+}
+
+// TODO: replace with a proper query builder
+export const buildSearchQueryWhereCondition = async ({
+  startTime,
+  endTime,
+  query,
+  propertyTypeMappingsModel,
+  teamId,
+}: {
+  startTime: number; // unix in ms
+  endTime: number; // unix in ms,
+  query: string;
+  propertyTypeMappingsModel: PropertyTypeMappingsModel;
+  teamId?: string;
+}) => {
+  const builder = new SearchQueryBuilder(query, propertyTypeMappingsModel);
+  builder.teamId = teamId;
+  return await builder.timestampInBetween(startTime, endTime).build();
+};

packages/api/src/config.ts

@@ -0,0 +1,35 @@
+import { version } from '../package.json';
+
+const env = process.env;
+
+export const CODE_VERSION = version;
+
+export const APP_TYPE = env.APP_TYPE as 'api' | 'aggregator' | 'scheduled-task';
+
+export const NODE_ENV = env.NODE_ENV as string;
+export const IS_PROD = NODE_ENV === 'production';
+export const IS_DEV = NODE_ENV === 'development';
+export const IS_CI = NODE_ENV === 'ci';
+
+export const PORT = Number.parseInt(env.PORT as string);
+export const SERVER_URL = env.SERVER_URL as string;
+export const FRONTEND_URL = env.FRONTEND_URL as string;
+export const COOKIE_DOMAIN = env.COOKIE_DOMAIN as string; // prod ONLY
+
+export const MONGO_URI = env.MONGO_URI as string;
+
+export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string;
+export const CLICKHOUSE_PASSWORD = env.CLICKHOUSE_PASSWORD as string;
+export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string;
+
+export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string;
+export const HYPERDX_INGESTOR_ENDPOINT =
+  env.HYPERDX_INGESTOR_ENDPOINT as string;
+
+export const EXPRESS_SESSION_SECRET = env.EXPRESS_SESSION_SECRET as string;
+
+export const REDIS_URL = env.REDIS_URL as string;
+
+export const MINER_API_URL = env.MINER_API_URL as string;
+
+export const OTEL_SERVICE_NAME = env.OTEL_SERVICE_NAME as string;

packages/api/src/controllers/__tests__/team.test.ts

@@ -0,0 +1,29 @@
+import { clearDBCollections, closeDB, connectDB } from '../../fixtures';
+import { createTeam, getTeam, getTeamByApiKey } from '../../controllers/team';
+
+describe('team controller', () => {
+  beforeAll(async () => {
+    await connectDB();
+  });
+
+  afterEach(async () => {
+    await clearDBCollections();
+  });
+
+  afterAll(async () => {
+    await closeDB();
+  });
+
+  it('createTeam + getTeam', async () => {
+    const team = await createTeam({ name: 'My Team' });
+
+    expect(team.name).toBe('My Team');
+
+    team.apiKey = 'apiKey';
+
+    await team.save();
+
+    expect(await getTeam(team._id)).toBeTruthy();
+    expect(await getTeamByApiKey('apiKey')).toBeTruthy();
+  });
+});

packages/api/src/controllers/team.ts

@@ -0,0 +1,34 @@
+import { v4 as uuidv4 } from 'uuid';
+
+import Team from '../models/team';
+
+import type { ObjectId } from '../models';
+
+export async function isTeamExisting() {
+  const teamCount = await Team.countDocuments({});
+  return teamCount > 0;
+}
+
+export async function createTeam({ name }: { name: string }) {
+  if (await isTeamExisting()) {
+    throw new Error('Team already exists');
+  }
+
+  const team = new Team({ name });
+
+  await team.save();
+
+  return team;
+}
+
+export function getTeam(id: string | ObjectId) {
+  return Team.findById(id);
+}
+
+export function getTeamByApiKey(apiKey: string) {
+  return Team.findOne({ apiKey });
+}
+
+export function rotateTeamApiKey(teamId: ObjectId) {
+  return Team.findByIdAndUpdate(teamId, { apiKey: uuidv4() }, { new: true });
+}

packages/api/src/controllers/user.ts

@@ -0,0 +1,22 @@
+import User from '../models/user';
+
+import type { ObjectId } from '../models';
+
+export function findUserById(id: string) {
+  return User.findById(id);
+}
+
+export function findUserByEmail(email: string) {
+  return User.findOne({ email });
+}
+
+export async function findUserByEmailInTeam(
+  email: string,
+  team: string | ObjectId,
+) {
+  return User.findOne({ email, team });
+}
+
+export function findUsersByTeam(team: string | ObjectId) {
+  return User.find({ team }).sort({ createdAt: 1 });
+}

packages/api/src/fixtures.ts

@@ -0,0 +1,65 @@
+import mongoose from 'mongoose';
+import request from 'supertest';
+
+import * as config from './config';
+import Server from './server';
+import { createTeam } from './controllers/team';
+import { mongooseConnection } from './models';
+
+export const connectDB = async () => {
+  if (!config.IS_CI) {
+    throw new Error('ONLY execute this in CI env 😈 !!!');
+  }
+  await mongoose.connect(config.MONGO_URI);
+};
+
+export const closeDB = async () => {
+  if (!config.IS_CI) {
+    throw new Error('ONLY execute this in CI env 😈 !!!');
+  }
+  await mongooseConnection.dropDatabase();
+};
+
+export const clearDBCollections = async () => {
+  if (!config.IS_CI) {
+    throw new Error('ONLY execute this in CI env 😈 !!!');
+  }
+  const collections = mongooseConnection.collections;
+  await Promise.all(
+    Object.values(collections).map(async collection => {
+      await collection.deleteMany({}); // an empty mongodb selector object ({}) must be passed as the filter argument
+    }),
+  );
+};
+
+// after connectDB
+export const initCiEnvs = async () => {
+  if (!config.IS_CI) {
+    throw new Error('ONLY execute this in CI env 😈 !!!');
+  }
+  // create a fake team with fake api key + setup gh integration
+  await createTeam({ name: 'Fake Team' });
+};
+
+class MockServer extends Server {
+  getHttpServer() {
+    return this.httpServer;
+  }
+
+  closeHttpServer() {
+    return new Promise<void>((resolve, reject) => {
+      this.httpServer.close(err => {
+        if (err) {
+          reject(err);
+          return;
+        }
+        resolve();
+      });
+    });
+  }
+}
+
+export const getServer = () => new MockServer();
+
+export const getAgent = (server: MockServer) =>
+  request.agent(server.getHttpServer());

packages/api/src/index.ts

@@ -0,0 +1,47 @@
+import { serializeError } from 'serialize-error';
+
+import * as config from './config';
+import Server from './server';
+import logger from './utils/logger';
+import { initCiEnvs } from './fixtures';
+import { isOperationalError } from './utils/errors';
+
+const server = new Server();
+
+process.on('uncaughtException', (err: Error) => {
+  logger.error(serializeError(err));
+
+  // FIXME: disable server restart until
+  // we make sure all expected exceptions are handled properly
+  if (config.IS_DEV && !isOperationalError(err)) {
+    process.exit(1);
+  }
+});
+
+process.on('unhandledRejection', (err: Error) => {
+  // TODO: do we want to throw here ?
+  logger.error(serializeError(err));
+});
+
+// graceful shutdown
+process.on('SIGTERM', () => {
+  logger.info('SIGTERM signal received.');
+
+  if (config.IS_DEV) {
+    logger.info('Http server is forced to stop immediately.');
+    process.exit(0);
+  }
+
+  server.stop();
+});
+
+server
+  .start()
+  .then(() => {
+    // TODO: a quick hack to work with e2e. We should do this in separate op
+    if (config.IS_CI) {
+      // place where we setup fake data for CI
+      return initCiEnvs();
+    }
+  })
+  .catch(e => logger.error(serializeError(e)));

packages/api/src/middleware/auth.ts

@@ -0,0 +1,76 @@
+import type { Request, Response, NextFunction } from 'express';
+
+import { serializeError } from 'serialize-error';
+import { setTraceAttributes } from '@hyperdx/node-opentelemetry';
+
+import * as config from '../config';
+import logger from '../utils/logger';
+
+import type { UserDocument } from '../models/user';
+
+declare global {
+  namespace Express {
+    interface User extends UserDocument {}
+  }
+}
+
+declare module 'express-session' {
+  interface Session {
+    messages: string[]; // Set by passport
+    passport: { user: string }; // Set by passport
+  }
+}
+
+export function redirectToDashboard(req: Request, res: Response) {
+  if (req?.user?.team) {
+    return res.redirect(`${config.FRONTEND_URL}/search`);
+  } else {
+    logger.error(
+      `Password login for user failed, user or team not found ${req?.user?._id}`,
+    );
+    res.redirect(`${config.FRONTEND_URL}/login?err=unknown`);
+  }
+}
+
+export function handleAuthError(
+  err: any,
+  req: Request,
+  res: Response,
+  next: NextFunction,
+) {
+  logger.debug({ message: 'Auth error', authErr: serializeError(err) });
+  if (res.headersSent) {
+    return next(err);
+  }
+
+  // Get the latest auth error message
+  const lastMessage = req.session.messages?.at(-1);
+  logger.debug(`Auth error last message: ${lastMessage}`);
+
+  const returnErr =
+    lastMessage === 'Password or username is incorrect'
+      ? 'authFail'
+      : lastMessage ===
+        'Authentication method password is not allowed by your team admin.'
+      ? 'passwordAuthNotAllowed'
+      : 'unknown';
+
+  res.redirect(`${config.FRONTEND_URL}/login?err=${returnErr}`);
+}
+
+export function isUserAuthenticated(
+  req: Request,
+  res: Response,
+  next: NextFunction,
+) {
+  if (req.isAuthenticated()) {
+    // set user id as trace attribute
+    setTraceAttributes({
+      userId: req.user?._id.toString(),
+      userEmail: req.user?.email,
+    });
+
+    return next();
+  }
+  res.sendStatus(401);
+}

packages/api/src/middleware/cors.ts

@@ -0,0 +1,6 @@
+import cors from 'cors';
+import { FRONTEND_URL } from '../config';
+
+export const noCors = cors();
+
+export default cors({ credentials: true, origin: FRONTEND_URL });

packages/api/src/middleware/error.ts

@@ -0,0 +1,29 @@
+import type { Request, Response, NextFunction } from 'express';
+
+import { serializeError } from 'serialize-error';
+
+import logger from '../utils/logger';
+import { BaseError, StatusCode, isOperationalError } from '../utils/errors';
+
+// WARNING: need to keep the 4th arg for express to identify it as an error-handling middleware function
+export const appErrorHandler = (
+  err: BaseError,
+  _: Request,
+  res: Response,
+  next: NextFunction,
+) => {
+  logger.error({
+    location: 'appErrorHandler',
+    error: serializeError(err),
+  });
+
+  const userFacingErrorMessage = isOperationalError(err)
+    ? err.message
+    : 'Something went wrong :(';
+
+  if (!res.headersSent) {
+    res
+      .status(err.statusCode ?? StatusCode.INTERNAL_SERVER)
+      .send(userFacingErrorMessage);
+  }
+};

packages/api/src/models/alert.ts

@@ -0,0 +1,87 @@
+import mongoose, { Schema } from 'mongoose';
+
+import type { ObjectId } from '.';
+
+export type AlertType = 'presence' | 'absence';
+
+export enum AlertState {
+  ALERT = 'ALERT',
+  DISABLED = 'DISABLED',
+  INSUFFICIENT_DATA = 'INSUFFICIENT_DATA',
+  OK = 'OK',
+}
+
+// follow 'ms' pkg formats
+export type AlertInterval =
+  | '1m'
+  | '5m'
+  | '15m'
+  | '30m'
+  | '1h'
+  | '6h'
+  | '12h'
+  | '1d';
+
+export type AlertChannel = {
+  type: 'webhook';
+  webhookId: string;
+};
+
+export interface IAlert {
+  _id: ObjectId;
+  channel: AlertChannel;
+  cron: string;
+  groupBy?: string;
+  interval: AlertInterval;
+  logView: ObjectId;
+  message?: string;
+  state: AlertState;
+  threshold: number;
+  timezone: string;
+  type: AlertType;
+}
+
+const AlertSchema = new Schema<IAlert>(
+  {
+    type: {
+      type: String,
+      required: true,
+    },
+    message: {
+      type: String,
+      required: false,
+    },
+    threshold: {
+      type: Number,
+      required: true,
+    },
+    interval: {
+      type: String,
+      required: true,
+    },
+    timezone: {
+      type: String,
+      required: true,
+    },
+    cron: {
+      type: String,
+      required: true,
+    },
+    channel: Schema.Types.Mixed, // slack, email, etc
+    logView: { type: mongoose.Schema.Types.ObjectId, ref: 'Alert' },
+    state: {
+      type: String,
+      enum: AlertState,
+      default: AlertState.OK,
+    },
+    groupBy: {
+      type: String,
+      required: false,
+    },
+  },
+  {
+    timestamps: true,
+  },
+);
+
+export default mongoose.model<IAlert>('Alert', AlertSchema);

packages/api/src/models/alertHistory.ts

@@ -0,0 +1,32 @@
+import mongoose, { Schema } from 'mongoose';
+import ms from 'ms';
+
+import type { ObjectId } from '.';
+
+export interface IAlertHistory {
+  alert: ObjectId;
+  counts: number;
+  createdAt: Date;
+}
+
+const AlertHistorySchema = new Schema<IAlertHistory>({
+  counts: {
+    type: Number,
+    default: 0,
+  },
+  createdAt: {
+    type: Date,
+    required: true,
+  },
+  alert: { type: mongoose.Schema.Types.ObjectId, ref: 'Alert' },
+});
+
+AlertHistorySchema.index(
+  { createdAt: 1 },
+  { expireAfterSeconds: ms('30d') / 1000 },
+);
+
+export default mongoose.model<IAlertHistory>(
+  'AlertHistory',
+  AlertHistorySchema,
+);

packages/api/src/models/dashboard.ts

@@ -0,0 +1,28 @@
+import mongoose, { Schema } from 'mongoose';
+
+import type { ObjectId } from '.';
+
+export interface IDashboard {
+  _id: ObjectId;
+  name: string;
+  query: string;
+  team: ObjectId;
+  charts: any[]; // TODO: Type this eventually
+}
+
+const DashboardSchema = new Schema<IDashboard>(
+  {
+    name: {
+      type: String,
+      required: true,
+    },
+    query: String,
+    team: { type: mongoose.Schema.Types.ObjectId, ref: 'Team' },
+    charts: { type: mongoose.Schema.Types.Mixed, required: true },
+  },
+  {
+    timestamps: true,
+  },
+);
+
+export default mongoose.model<IDashboard>('Dashboard', DashboardSchema);

packages/api/src/models/index.ts

@@ -0,0 +1,44 @@
+import mongoose from 'mongoose';
+
+import * as config from '../config';
+import logger from '../utils/logger';
+
+export type ObjectId = mongoose.Types.ObjectId;
+
+// set flags
+mongoose.set('strictQuery', true);
+
+// Allow empty strings to be set to required fields
+// https://github.com/Automattic/mongoose/issues/7150
+// ex. query in logview can be empty
+mongoose.Schema.Types.String.checkRequired(v => v != null);
+
+// connection events handlers
+mongoose.connection.on('connected', () => {
+  logger.info('Connection established to MongoDB');
+});
+
+mongoose.connection.on('disconnected', () => {
+  logger.info('Lost connection to MongoDB server');
+});
+
+mongoose.connection.on('error', () => {
+  logger.error('Could not connect to MongoDB');
+});
+
+mongoose.connection.on('reconnected', () => {
+  logger.error('Reconnected to MongoDB');
+});
+
+mongoose.connection.on('reconnectFailed', () => {
+  logger.error('Failed to reconnect to MongoDB');
+});
+
+export const connectDB = async () => {
+  await mongoose.connect(config.MONGO_URI, {
+    heartbeatFrequencyMS: 10000, // retry failed heartbeats
+    maxPoolSize: 100, // 5 nodes -> max 1000 connections
+  });
+};
+
+export const mongooseConnection = mongoose.connection;

packages/api/src/models/logView.ts

@@ -0,0 +1,31 @@
+import mongoose, { Schema } from 'mongoose';
+
+import type { ObjectId } from '.';
+
+export interface ILogView {
+  _id: ObjectId;
+  creator: ObjectId;
+  name: string;
+  query: string;
+  team: ObjectId;
+}
+
+const LogViewSchema = new Schema<ILogView>(
+  {
+    query: {
+      type: String,
+      required: true,
+    },
+    name: {
+      type: String,
+      required: true,
+    },
+    team: { type: mongoose.Schema.Types.ObjectId, ref: 'Team' },
+    creator: { type: mongoose.Schema.Types.ObjectId, ref: 'User' },
+  },
+  {
+    timestamps: true,
+  },
+);
+
+export default mongoose.model<ILogView>('LogView', LogViewSchema);

packages/api/src/models/team.ts

@@ -0,0 +1,39 @@
+import mongoose, { Schema } from 'mongoose';
+import { v4 as uuidv4 } from 'uuid';
+
+type ObjectId = mongoose.Types.ObjectId;
+
+export interface ITeam {
+  _id: ObjectId;
+  name: string;
+  logStreamTableVersion?: number;
+  allowedAuthMethods?: 'password'[];
+  apiKey: string;
+  hookId: string;
+}
+
+export default mongoose.model<ITeam>(
+  'Team',
+  new Schema<ITeam>(
+    {
+      name: String,
+      allowedAuthMethods: [String],
+      hookId: {
+        type: String,
+        default: function genUUID() {
+          return uuidv4();
+        },
+      },
+      apiKey: {
+        type: String,
+        default: function genUUID() {
+          return uuidv4();
+        },
+      },
+      logStreamTableVersion: Number,
+    },
+    {
+      timestamps: true,
+    },
+  ),
+);

packages/api/src/models/teamInvite.ts

@@ -0,0 +1,36 @@
+import mongoose, { Schema } from 'mongoose';
+
+export interface ITeamInvite {
+  createdAt: Date;
+  email: string;
+  name?: string;
+  teamId: string;
+  token: string;
+  updatedAt: Date;
+}
+
+const TeamInviteSchema = new Schema(
+  {
+    teamId: {
+      type: Schema.Types.ObjectId,
+      ref: 'Team',
+      required: true,
+    },
+    name: String,
+    email: {
+      type: String,
+      required: true,
+    },
+    token: {
+      type: String,
+      required: true,
+    },
+  },
+  {
+    timestamps: true,
+  },
+);
+
+TeamInviteSchema.index({ teamId: 1, email: 1 }, { unique: true });
+
+export default mongoose.model<ITeamInvite>('TeamInvite', TeamInviteSchema);

packages/api/src/models/user.ts

@@ -0,0 +1,44 @@
+import mongoose, { Schema } from 'mongoose';
+import { v4 as uuidv4 } from 'uuid';
+// @ts-ignore don't install the @types for this package, as it conflicts with mongoose
+import passportLocalMongoose from 'passport-local-mongoose';
+
+type ObjectId = mongoose.Types.ObjectId;
+
+export interface IUser {
+  _id: ObjectId;
+  createdAt: Date;
+  email: string;
+  name: string;
+  team: ObjectId;
+}
+
+export type UserDocument = mongoose.HydratedDocument<IUser>;
+
+const UserSchema = new Schema(
+  {
+    name: String,
+    email: {
+      type: String,
+      required: true,
+    },
+    team: { type: mongoose.Schema.Types.ObjectId, ref: 'Team' },
+  },
+  {
+    timestamps: true,
+  },
+);
+
+UserSchema.virtual('hasPasswordAuth').get(function (this: IUser) {
+  return true;
+});
+
+UserSchema.plugin(passportLocalMongoose, {
+  usernameField: 'email',
+  usernameLowerCase: true,
+  usernameCaseInsensitive: true,
+});
+
+UserSchema.index({ email: 1 }, { unique: true });
+
+export default mongoose.model<IUser>('User', UserSchema);

packages/api/src/models/webhook.ts

@@ -0,0 +1,26 @@
+import mongoose, { Schema } from 'mongoose';
+
+import { ObjectId } from 'mongodb';
+
+export interface IWebhook {
+  _id: ObjectId;
+  createdAt: Date;
+  name: string;
+  service: string;
+  team: ObjectId;
+  updatedAt: Date;
+  url: string;
+}
+
+export default mongoose.model<IWebhook>(
+  'Webhook',
+  new Schema<IWebhook>(
+    {
+      team: { type: Schema.Types.ObjectId, ref: 'Team' },
+      service: String,
+      name: String,
+      url: String,
+    },
+    { timestamps: true },
+  ),
+);

packages/api/src/routers/aggregator/index.ts

@@ -0,0 +1,5 @@
+import rootRouter from './root';
+
+export default {
+  rootRouter,
+};

packages/api/src/routers/aggregator/root.ts

@@ -0,0 +1,127 @@
+import express from 'express';
+import groupBy from 'lodash/groupBy';
+
+import * as config from '../../config';
+import logger from '../../utils/logger';
+import {
+  bulkInsertRrwebEvents,
+  bulkInsertTeamLogStream,
+  bulkInsertTeamMetricStream,
+} from '../../clickhouse';
+import {
+  extractApiKey,
+  vectorLogParser,
+  vectorMetricParser,
+  vectorRrwebParser,
+} from '../../utils/logParser';
+import { getTeamByApiKey } from '../../controllers/team';
+
+import type { VectorLog, VectorMetric } from '../../utils/logParser';
+
+const router = express.Router();
+
+const bulkInsert = async (
+  hdxTelemetry: string | undefined,
+  apiKey: string,
+  data: (VectorLog | VectorMetric)[],
+) => {
+  const team = await getTeamByApiKey(apiKey);
+  if (team) {
+    switch (hdxTelemetry) {
+      case 'metric':
+        await bulkInsertTeamMetricStream(
+          vectorMetricParser.parse(data as VectorMetric[]),
+        );
+        break;
+      default: {
+        const rrwebEvents = [];
+        const logs = [];
+        for (const log of data) {
+          if (log.hdx_platform === 'rrweb') {
+            rrwebEvents.push(log);
+          } else {
+            logs.push(log);
+          }
+        }
+        const promises = [
+          bulkInsertTeamLogStream(
+            team.logStreamTableVersion,
+            team._id.toString(),
+            vectorLogParser.parse(logs as VectorLog[]),
+          ),
+        ];
+        if (rrwebEvents.length > 0) {
+          promises.push(
+            bulkInsertRrwebEvents(
+              vectorRrwebParser.parse(rrwebEvents as VectorLog[]),
+            ),
+          );
+        }
+        await Promise.all(promises);
+        break;
+      }
+    }
+  }
+};
+
+router.get('/health', (_, res) => {
+  res.send({ data: 'OK', version: config.CODE_VERSION });
+});
+
+// bulk insert logs
+router.post('/', async (req, res, next) => {
+  const { telemetry } = req.query;
+  const hdxTelemetry = (telemetry ?? 'log') as string;
+  try {
+    const logs: (VectorLog | VectorMetric)[] = req.body;
+    if (!Array.isArray(logs)) {
+      return res.sendStatus(400);
+    }
+    // TODO: move this to the end of the request so vector will buffer the logs
+    // Need to check request.timeout_secs config
+    res.sendStatus(200);
+
+    logger.info({
+      message: `Received ${hdxTelemetry}`,
+      size: JSON.stringify(logs).length,
+      n: logs.length,
+    });
+
+    const filteredLogs = logs
+      .map(log => ({
+        ...log,
+        hdx_apiKey: extractApiKey(log),
+      }))
+      // check hdx_platform values ?
+      .filter(log => log.hdx_platform && log.hdx_apiKey);
+
+    if (logs.length - filteredLogs.length > 0) {
+      // TEMP: DEBUGGING (remove later)
+      const droppedLogs = logs
+        .map(log => ({
+          ...log,
+          hdx_apiKey: extractApiKey(log),
+        }))
+        .filter(log => !log.hdx_platform || !log.hdx_apiKey);
+      logger.info({
+        message: `Dropped ${hdxTelemetry}`,
+        n: filteredLogs.length,
+        diff: logs.length - filteredLogs.length,
+        droppedLogs,
+      });
+    }
+
+    if (filteredLogs.length > 0) {
+      const groupedLogs = groupBy(filteredLogs, 'hdx_apiKey');
+      await Promise.all(
+        Object.entries(groupedLogs).map(([apiKey, logs]) =>
+          bulkInsert(hdxTelemetry, apiKey, logs),
+        ),
+      );
+    }
+  } catch (e) {
+    next(e);
+  }
+});
+
+export default router;

packages/api/src/routers/api/__tests__/team.test.ts

@@ -0,0 +1,85 @@
+import _ from 'lodash';
+
+import {
+  clearDBCollections,
+  closeDB,
+  getAgent,
+  getServer,
+} from '../../../fixtures';
+import { getTeam } from '../../../controllers/team';
+import { findUserByEmail } from '../../../controllers/user';
+
+describe('team router', () => {
+  const server = getServer();
+
+  const login = async () => {
+    const agent = getAgent(server);
+
+    await agent
+      .post('/register/password')
+      .send({
+        email: 'fake@deploysentinel.com',
+        password: 'tacocat1234',
+      })
+      .expect(302);
+
+    const user = await findUserByEmail('fake@deploysentinel.com');
+    const team = await getTeam(user?.team as any);
+
+    if (team === null || user === null) {
+      throw Error('team or user not found');
+    }
+
+    await user.save();
+
+    // login app
+    await agent
+      .post('/login/password')
+      .send({
+        email: 'fake@deploysentinel.com',
+        password: 'tacocat1234',
+      })
+      .expect(302);
+
+    return {
+      agent,
+      team,
+      user,
+    };
+  };
+
+  beforeAll(async () => {
+    await server.start();
+  });
+
+  afterEach(async () => {
+    await clearDBCollections();
+  });
+
+  afterAll(async () => {
+    await server.closeHttpServer();
+    await closeDB();
+  });
+
+  it('GET /team', async () => {
+    const { agent } = await login();
+
+    const resp = await agent.get('/team').expect(200);
+
+    expect(_.omit(resp.body, ['_id', 'apiKey'])).toMatchInlineSnapshot(`
+Object {
+  "allowedAuthMethods": Array [],
+  "name": "fake@deploysentinel.com's Team",
+  "teamInvites": Array [],
+  "users": Array [
+    Object {
+      "email": "fake@deploysentinel.com",
+      "hasPasswordAuth": true,
+      "isCurrentUser": true,
+      "name": "fake@deploysentinel.com",
+    },
+  ],
+}
+`);
+  });
+});

packages/api/src/routers/api/alerts.ts

@@ -0,0 +1,223 @@
+import express from 'express';
+import ms from 'ms';
+import { getHours, getMinutes } from 'date-fns';
+
+import Alert, {
+  AlertChannel,
+  AlertInterval,
+  AlertType,
+} from '../../models/alert';
+import * as clickhouse from '../../clickhouse';
+import { SQLSerializer } from '../../clickhouse/searchQueryParser';
+import { getTeam } from '../../controllers/team';
+import { isUserAuthenticated } from '../../middleware/auth';
+
+const router = express.Router();
+
+const getCron = (interval: AlertInterval) => {
+  const now = new Date();
+  const nowMins = getMinutes(now);
+  const nowHours = getHours(now);
+
+  switch (interval) {
+    case '1m':
+      return '* * * * *';
+    case '5m':
+      return '*/5 * * * *';
+    case '15m':
+      return '*/15 * * * *';
+    case '30m':
+      return '*/30 * * * *';
+    case '1h':
+      return `${nowMins} * * * *`;
+    case '6h':
+      return `${nowMins} */6 * * *`;
+    case '12h':
+      return `${nowMins} */12 * * *`;
+    case '1d':
+      return `${nowMins} ${nowHours} * * *`;
+  }
+};
+
+const createAlert = async ({
+  channel,
+  groupBy,
+  interval,
+  logViewId,
+  threshold,
+  type,
+}: {
+  channel: AlertChannel;
+  groupBy?: string;
+  interval: AlertInterval;
+  logViewId: string;
+  threshold: number;
+  type: AlertType;
+}) => {
+  return new Alert({
+    channel,
+    cron: getCron(interval),
+    groupBy,
+    interval,
+    logView: logViewId,
+    threshold,
+    timezone: 'UTC', // TODO: support different timezone
+    type,
+  }).save();
+};
+
+// create an update alert function based off of the above create alert function
+const updateAlert = async ({
+  channel,
+  groupBy,
+  id,
+  interval,
+  logViewId,
+  threshold,
+  type,
+}: {
+  channel: AlertChannel;
+  groupBy?: string;
+  id: string;
+  interval: AlertInterval;
+  logViewId: string;
+  threshold: number;
+  type: AlertType;
+}) => {
+  return Alert.findByIdAndUpdate(
+    id,
+    {
+      channel,
+      cron: getCron(interval),
+      groupBy: groupBy ?? null,
+      interval,
+      logView: logViewId,
+      threshold,
+      timezone: 'UTC', // TODO: support different timezone
+      type,
+    },
+    {
+      returnDocument: 'after',
+    },
+  );
+};
+
+router.post('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { channel, groupBy, interval, logViewId, threshold, type } = req.body;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (!channel || !threshold || !interval || !type) {
+      return res.sendStatus(400);
+    }
+    if (!['slack', 'email', 'pagerduty', 'webhook'].includes(channel.type)) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    // validate groupBy property
+    if (groupBy) {
+      const nowInMs = Date.now();
+      const propertyTypeMappingsModel =
+        await clickhouse.buildLogsPropertyTypeMappingsModel(
+          team.logStreamTableVersion,
+          teamId.toString(),
+          nowInMs - ms('1d'),
+          nowInMs,
+        );
+      const serializer = new SQLSerializer(propertyTypeMappingsModel);
+      const { found } = await serializer.getColumnForField(groupBy);
+      if (!found) {
+        return res.sendStatus(400);
+      }
+    }
+
+    res.json({
+      data: await createAlert({
+        channel,
+        groupBy,
+        interval,
+        logViewId,
+        threshold,
+        type,
+      }),
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.put('/:id', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { id: alertId } = req.params;
+    const { channel, interval, logViewId, threshold, type, groupBy } = req.body;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (!channel || !threshold || !interval || !type || !alertId) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    // validate groupBy property
+    if (groupBy) {
+      const nowInMs = Date.now();
+      const propertyTypeMappingsModel =
+        await clickhouse.buildLogsPropertyTypeMappingsModel(
+          team.logStreamTableVersion,
+          teamId.toString(),
+          nowInMs - ms('1d'),
+          nowInMs,
+        );
+      const serializer = new SQLSerializer(propertyTypeMappingsModel);
+      const { found } = await serializer.getColumnForField(groupBy);
+      if (!found) {
+        return res.sendStatus(400);
+      }
+    }
+
+    res.json({
+      data: await updateAlert({
+        channel,
+        groupBy,
+        id: alertId,
+        interval,
+        logViewId,
+        threshold,
+        type,
+      }),
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.delete('/:id', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { id: alertId } = req.params;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (!alertId) {
+      return res.sendStatus(400);
+    }
+    await Alert.findByIdAndDelete(alertId);
+    res.sendStatus(200);
+  } catch (e) {
+    next(e);
+  }
+});
+
+export default router;

packages/api/src/routers/api/dashboards.ts

@@ -0,0 +1,156 @@
+import express from 'express';
+
+import Dashboard from '../../models/dashboard';
+import { isUserAuthenticated } from '../../middleware/auth';
+import { validateRequest } from 'zod-express-middleware';
+import { z } from 'zod';
+
+// create routes that will get and update dashboards
+const router = express.Router();
+
+const zChart = z.object({
+  id: z.string(),
+  name: z.string(),
+  x: z.number(),
+  y: z.number(),
+  w: z.number(),
+  h: z.number(),
+  series: z.array(
+    // We can't do a strict validation here since mongo and the frontend
+    // have a bug where chart types will not delete extraneous properties
+    // when attempting to save.
+    z.object({
+      type: z.enum([
+        'time',
+        'histogram',
+        'search',
+        'number',
+        'table',
+        'markdown',
+      ]),
+      table: z.string().optional(),
+      aggFn: z.string().optional(), // TODO: Replace with the actual AggFn schema
+      field: z.union([z.string(), z.undefined()]).optional(),
+      where: z.string().optional(),
+      groupBy: z.array(z.string()).optional(),
+      sortOrder: z.union([z.literal('desc'), z.literal('asc')]).optional(),
+      content: z.string().optional(),
+    }),
+  ),
+});
+
+router.get('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+
+    const dashboards = await Dashboard.find(
+      { team: teamId },
+      { _id: 1, name: 1, createdAt: 1, updatedAt: 1, charts: 1, query: 1 },
+    ).sort({ name: -1 });
+
+    res.json({
+      data: dashboards,
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.post(
+  '/',
+  isUserAuthenticated,
+  validateRequest({
+    body: z.object({
+      name: z.string(),
+      charts: z.array(zChart),
+      query: z.string(),
+    }),
+  }),
+  async (req, res, next) => {
+    try {
+      const teamId = req.user?.team;
+      if (teamId == null) {
+        return res.sendStatus(403);
+      }
+
+      const { name, charts, query } = req.body ?? {};
+      // Create new dashboard from name and charts
+      const newDashboard = await new Dashboard({
+        name,
+        charts,
+        query,
+        team: teamId,
+      }).save();
+
+      res.json({
+        data: newDashboard,
+      });
+    } catch (e) {
+      next(e);
+    }
+  },
+);
+
+router.put(
+  '/:id',
+  isUserAuthenticated,
+  validateRequest({
+    body: z.object({
+      name: z.string(),
+      charts: z.array(zChart),
+      query: z.string(),
+    }),
+  }),
+  async (req, res, next) => {
+    try {
+      const teamId = req.user?.team;
+      const { id: dashboardId } = req.params;
+      if (teamId == null) {
+        return res.sendStatus(403);
+      }
+      if (!dashboardId) {
+        return res.sendStatus(400);
+      }
+
+      const { name, charts, query } = req.body ?? {};
+      // Update dashboard from name and charts
+      const updatedDashboard = await Dashboard.findByIdAndUpdate(
+        dashboardId,
+        {
+          name,
+          charts,
+          query,
+        },
+        { new: true },
+      );
+
+      res.json({
+        data: updatedDashboard,
+      });
+    } catch (e) {
+      next(e);
+    }
+  },
+);
+
+router.delete('/:id', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { id: dashboardId } = req.params;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (!dashboardId) {
+      return res.sendStatus(400);
+    }
+    await Dashboard.findByIdAndDelete(dashboardId);
+    res.json({});
+  } catch (e) {
+    next(e);
+  }
+});
+
+export default router;

packages/api/src/routers/api/index.ts

@@ -0,0 +1,21 @@
+import alertsRouter from './alerts';
+import dashboardRouter from './dashboards';
+import logViewsRouter from './logViews';
+import logsRouter from './logs';
+import metricsRouter from './metrics';
+import rootRouter from './root';
+import sessionsRouter from './sessions';
+import teamRouter from './team';
+import webhooksRouter from './webhooks';
+
+export default {
+  alertsRouter,
+  dashboardRouter,
+  logViewsRouter,
+  logsRouter,
+  metricsRouter,
+  rootRouter,
+  sessionsRouter,
+  teamRouter,
+  webhooksRouter,
+};

packages/api/src/routers/api/logViews.ts

@@ -0,0 +1,109 @@
+import express from 'express';
+
+import Alert from '../../models/alert';
+import LogView from '../../models/logView';
+import { isUserAuthenticated } from '../../middleware/auth';
+
+const router = express.Router();
+
+router.post('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const userId = req.user?._id;
+    const { query, name } = req.body;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (query == null || !name) {
+      return res.sendStatus(400);
+    }
+    const logView = await new LogView({
+      name,
+      query: `${query}`,
+      team: teamId,
+      creator: userId,
+    }).save();
+
+    res.json({
+      data: logView,
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.get('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    const logViews = await LogView.find(
+      { team: teamId },
+      {
+        name: 1,
+        query: 1,
+        createdAt: 1,
+        updatedAt: 1,
+      },
+    ).sort({ createdAt: -1 });
+    const allAlerts = await Promise.all(
+      logViews.map(lv => Alert.find({ logView: lv._id }, { __v: 0 })),
+    );
+    res.json({
+      data: logViews.map((lv, idx) => ({
+        ...lv.toJSON(),
+        alerts: allAlerts[idx],
+      })),
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.patch('/:id', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { id: logViewId } = req.params;
+    const { query } = req.body;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (!logViewId || !query) {
+      return res.sendStatus(400);
+    }
+    const logView = await LogView.findByIdAndUpdate(
+      logViewId,
+      {
+        query,
+      },
+      { new: true },
+    );
+    res.json({
+      data: logView,
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.delete('/:id', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { id: logViewId } = req.params;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (!logViewId) {
+      return res.sendStatus(400);
+    }
+    // delete all alerts
+    await Alert.deleteMany({ logView: logViewId });
+    await LogView.findByIdAndDelete(logViewId);
+    res.sendStatus(200);
+  } catch (e) {
+    next(e);
+  }
+});
+
+export default router;

packages/api/src/routers/api/logs.ts

@@ -0,0 +1,527 @@
+import express from 'express';
+import ms from 'ms';
+import opentelemetry, { SpanStatusCode } from '@opentelemetry/api';
+import { isNumber, omit, parseInt } from 'lodash';
+import { serializeError } from 'serialize-error';
+import { validateRequest } from 'zod-express-middleware';
+import { z } from 'zod';
+
+import * as clickhouse from '../../clickhouse';
+import { isUserAuthenticated } from '../../middleware/auth';
+import logger from '../../utils/logger';
+import { LimitedSizeQueue } from '../../utils/queue';
+import { customColumnMapType } from '../../clickhouse/searchQueryParser';
+import { getLogsPatterns } from '../../utils/miner';
+import { getTeam } from '../../controllers/team';
+
+const router = express.Router();
+
+router.get('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { endTime, offset, q, startTime, order, limit } = req.query;
+    let { extraFields } = req.query;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+
+    if (extraFields == null) {
+      extraFields = [];
+    }
+
+    if (
+      !Array.isArray(extraFields) ||
+      (extraFields?.length > 0 && typeof extraFields[0] != 'string')
+    ) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    const MAX_LIMIT = 4000;
+
+    res.json(
+      await clickhouse.getLogBatch({
+        extraFields: extraFields as string[],
+        endTime: parseInt(endTime as string),
+        limit: Number.isInteger(Number.parseInt(`${limit}`))
+          ? Math.min(MAX_LIMIT, Number.parseInt(`${limit}`))
+          : 100,
+        offset: parseInt(offset as string),
+        q: q as string,
+        order: order === 'null' ? null : order === 'asc' ? 'asc' : 'desc',
+        startTime: parseInt(startTime as string),
+        tableVersion: team.logStreamTableVersion,
+        teamId: teamId.toString(),
+      }),
+    );
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+
+    next(e);
+  }
+});
+
+router.get('/patterns', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { endTime, q, startTime, sampleRate } = req.query;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (!endTime || !startTime) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    const MAX_LOG_BODY_LENGTH = 512;
+    const MAX_LOG_GROUPS = 1e4;
+    const MAX_SAMPLES = 50;
+    const TOTAL_TRENDS_BUCKETS = 15;
+    const SAMPLE_RATE = sampleRate ? parseFloat(sampleRate as string) : 1; // TODO: compute this dynamically
+    const scaleSampleCounts = (count: number) =>
+      Math.round(count / SAMPLE_RATE);
+
+    const msRange = parseInt(endTime as string) - parseInt(startTime as string);
+    const interval = clickhouse.msRangeToHistogramInterval(
+      msRange,
+      TOTAL_TRENDS_BUCKETS,
+    );
+
+    const logs = await clickhouse.getLogBatchGroupedByBody({
+      bodyMaxLength: MAX_LOG_BODY_LENGTH,
+      endTime: parseInt(endTime as string),
+      interval,
+      limit: MAX_LOG_GROUPS,
+      q: q as string,
+      sampleRate: SAMPLE_RATE,
+      startTime: parseInt(startTime as string),
+      tableVersion: team.logStreamTableVersion,
+      teamId: teamId.toString(),
+    });
+
+    if (logs.data.length === 0) {
+      return res.json({ data: [] });
+    }
+
+    // use the 1st id as the representative id
+    const lines = logs.data.map(log => [log.ids[0], log.body]);
+    // TODO: separate patterns by service
+    const logsPatternsData = await getLogsPatterns(teamId.toString(), lines);
+    type Sample = {
+      id: string;
+      body: string;
+      timestamp: string;
+      sort_key: string;
+    };
+    const result: Record<
+      string,
+      {
+        count: number;
+        level: string;
+        patternId: string;
+        samples: LimitedSizeQueue<Sample>;
+        service: string;
+        trends: Record<string, number>;
+      }
+    > = {};
+    for (const log of logs.data) {
+      const patternId = logsPatternsData.result[log.ids[0]];
+      if (patternId) {
+        const pattern = logsPatternsData.patterns[patternId];
+        if (!(pattern in result)) {
+          result[pattern] = {
+            count: 0,
+            level: log.level,
+            patternId,
+            samples: new LimitedSizeQueue<Sample>(MAX_SAMPLES),
+            service: log.service, // FIXME: multiple services might share the same pattern
+            trends: {},
+          };
+        }
+
+        for (const [idx, timestamp] of log.timestamps.entries()) {
+          result[pattern].samples.enqueue({
+            body: log.body,
+            id: log.ids[idx],
+            sort_key: log.sort_keys[idx],
+            timestamp,
+          });
+
+          // compute trends
+          const bucket = log.buckets[idx];
+          if (!(bucket in result[pattern].trends)) {
+            result[pattern].trends[bucket] = 0;
+          }
+          result[pattern].trends[bucket] += scaleSampleCounts(1);
+        }
+        result[pattern].count += scaleSampleCounts(parseInt(log.lines_count));
+      }
+    }
+
+    res.json({
+      data: Object.entries(result)
+        .map(([pattern, meta]) => ({
+          count: meta.count,
+          level: meta.level,
+          pattern,
+          id: meta.patternId,
+          samples: meta.samples
+            .toArray()
+            .sort(
+              (a, b) =>
+                new Date(b.timestamp).getTime() -
+                new Date(a.timestamp).getTime(),
+            ),
+          service: meta.service,
+          trends: {
+            granularity: interval,
+            data: Object.entries(meta.trends)
+              .map(([bucket, count]) => ({
+                bucket,
+                count,
+              }))
+              .sort(
+                (a, b) =>
+                  new Date(a.bucket).getTime() - new Date(b.bucket).getTime(),
+              ),
+          },
+        }))
+        .sort((a, b) => b.count - a.count),
+    });
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+
+    next(e);
+  }
+});
+
+router.get('/stream', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { endTime, offset, q, startTime, order, limit } = req.query;
+    let { extraFields } = req.query;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+
+    if (extraFields == null) {
+      extraFields = [];
+    }
+
+    if (
+      !Array.isArray(extraFields) ||
+      (extraFields?.length > 0 && typeof extraFields[0] != 'string')
+    ) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    const MAX_LIMIT = 4000;
+
+    res.setHeader('Cache-Control', 'no-cache');
+    res.setHeader('Content-Type', 'text/event-stream');
+    res.setHeader('Connection', 'keep-alive');
+    res.flushHeaders(); // flush the headers to establish SSE with client
+
+    // TODO: verify query
+    const stream = await clickhouse.getLogStream({
+      extraFields: extraFields as string[],
+      endTime: parseInt(endTime as string),
+      limit: Number.isInteger(Number.parseInt(`${limit}`))
+        ? Math.min(MAX_LIMIT, Number.parseInt(`${limit}`))
+        : 100,
+      offset: parseInt(offset as string),
+      order: order === 'null' ? null : order === 'asc' ? 'asc' : 'desc',
+      q: q as string,
+      startTime: parseInt(startTime as string),
+      tableVersion: team.logStreamTableVersion,
+      teamId: teamId.toString(),
+    });
+
+    let resultCount = 0;
+
+    if (stream == null) {
+      logger.info('No results found for query');
+
+      res.write('event: end\ndata:\n\n');
+      res.end();
+    } else {
+      stream.on('data', (rows: any[]) => {
+        resultCount += rows.length;
+        logger.info(`Sending ${rows.length} rows`);
+
+        res.write(`${rows.map(row => `data: ${row.text}`).join('\n')}\n\n`);
+        res.flush();
+      });
+      stream.on('end', () => {
+        res.write('event: end\ndata:\n\n');
+        res.end();
+      });
+    }
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+    // WARNING: no need to call next(e) here, as the stream will be closed
+    logger.error({
+      message: 'Error streaming logs',
+      error: serializeError(e),
+      teamId: req.user?.team,
+      query: req.query,
+    });
+    res.end();
+  }
+});
+
+router.get(
+  '/propertyTypeMappings',
+  isUserAuthenticated,
+  async (req, res, next) => {
+    try {
+      const teamId = req.user?.team;
+      if (teamId == null) {
+        return res.sendStatus(403);
+      }
+
+      const team = await getTeam(teamId);
+      if (team == null) {
+        return res.sendStatus(403);
+      }
+
+      const nowInMs = Date.now();
+      const propertyTypeMappingsModel =
+        await clickhouse.buildLogsPropertyTypeMappingsModel(
+          team.logStreamTableVersion,
+          teamId.toString(),
+          nowInMs - ms('1d'),
+          nowInMs,
+        );
+
+      res.json({
+        data: [
+          ...propertyTypeMappingsModel.currentPropertyTypeMappings,
+          ...Object.entries(omit(customColumnMapType, ['<implicit>'])),
+        ],
+      });
+    } catch (e) {
+      const span = opentelemetry.trace.getActiveSpan();
+      span.recordException(e as Error);
+      span.setStatus({ code: SpanStatusCode.ERROR });
+      next(e);
+    }
+  },
+);
+
+router.get('/chart/histogram', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { endTime, field, q, startTime } = req.query;
+    if (teamId == null || typeof field !== 'string' || field == '') {
+      return res.sendStatus(403);
+    }
+    const startTimeNum = parseInt(startTime as string);
+    const endTimeNum = parseInt(endTime as string);
+    if (!isNumber(startTimeNum) || !isNumber(endTimeNum)) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    res.json(
+      await clickhouse.getChartHistogram({
+        bins: 20,
+        endTime: endTimeNum,
+        field: field as string,
+        q: q as string,
+        startTime: startTimeNum,
+        tableVersion: team.logStreamTableVersion,
+        teamId: teamId.toString(),
+      }),
+    );
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+    next(e);
+  }
+});
+
+router.get(
+  '/chart',
+  isUserAuthenticated,
+  validateRequest({
+    query: z.object({
+      aggFn: z.nativeEnum(clickhouse.AggFn),
+      endTime: z.string(),
+      field: z.string().optional(),
+      granularity: z.nativeEnum(clickhouse.Granularity).optional(),
+      groupBy: z.string().optional(),
+      q: z.string().optional(),
+      sortOrder: z.enum(['asc', 'desc']).optional(),
+      startTime: z.string(),
+    }),
+  }),
+  async (req, res, next) => {
+    try {
+      const teamId = req.user?.team;
+      const {
+        aggFn,
+        endTime,
+        field,
+        granularity,
+        groupBy,
+        q,
+        startTime,
+        sortOrder,
+      } = req.query;
+      if (teamId == null) {
+        return res.sendStatus(403);
+      }
+      const startTimeNum = parseInt(startTime);
+      const endTimeNum = parseInt(endTime);
+      if (!isNumber(startTimeNum) || !isNumber(endTimeNum)) {
+        return res.sendStatus(400);
+      }
+
+      const team = await getTeam(teamId);
+      if (team == null) {
+        return res.sendStatus(403);
+      }
+
+      const propertyTypeMappingsModel =
+        await clickhouse.buildLogsPropertyTypeMappingsModel(
+          team.logStreamTableVersion,
+          teamId.toString(),
+          startTimeNum,
+          endTimeNum,
+        );
+
+      // TODO: hacky way to make sure the cache is update to date
+      if (
+        !clickhouse.doesLogsPropertyExist(field, propertyTypeMappingsModel) ||
+        !clickhouse.doesLogsPropertyExist(groupBy, propertyTypeMappingsModel)
+      ) {
+        logger.warn({
+          message: `getChart: Property type mappings cache is out of date (${field}, ${groupBy}})`,
+        });
+        await propertyTypeMappingsModel.refresh();
+      }
+
+      // TODO: expose this to the frontend
+      const MAX_NUM_GROUPS = 20;
+
+      res.json(
+        await clickhouse.getLogsChart({
+          aggFn,
+          endTime: endTimeNum,
+          field,
+          granularity,
+          groupBy,
+          maxNumGroups: MAX_NUM_GROUPS,
+          propertyTypeMappingsModel,
+          q,
+          sortOrder,
+          startTime: startTimeNum,
+          tableVersion: team.logStreamTableVersion,
+          teamId: teamId.toString(),
+        }),
+      );
+    } catch (e) {
+      const span = opentelemetry.trace.getActiveSpan();
+      span.recordException(e as Error);
+      span.setStatus({ code: SpanStatusCode.ERROR });
+      next(e);
+    }
+  },
+);
+
+router.get('/histogram', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { endTime, q, startTime } = req.query;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    const startTimeNum = parseInt(startTime as string);
+    const endTimeNum = parseInt(endTime as string);
+    if (!isNumber(startTimeNum) || !isNumber(endTimeNum)) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    res.json(
+      await clickhouse.getHistogram(
+        team.logStreamTableVersion,
+        teamId.toString(),
+        q as string,
+        startTimeNum,
+        endTimeNum,
+      ),
+    );
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+
+    next(e);
+  }
+});
+
+router.get('/:id', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const logId = req.params.id;
+    const { sortKey } = req.query;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    if (!sortKey) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    res.json(
+      await clickhouse.getLogById(
+        team.logStreamTableVersion,
+        teamId.toString(),
+        sortKey as string,
+        logId,
+      ),
+    );
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+
+    next(e);
+  }
+});
+
+export default router;

packages/api/src/routers/api/metrics.ts

@@ -0,0 +1,68 @@
+import express from 'express';
+import opentelemetry, { SpanStatusCode } from '@opentelemetry/api';
+import { isNumber, parseInt } from 'lodash';
+
+import * as clickhouse from '../../clickhouse';
+import { isUserAuthenticated } from '../../middleware/auth';
+
+const router = express.Router();
+
+router.get('/tags', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    // TODO: use cache
+    res.json(await clickhouse.getMetricsTags(teamId.toString()));
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+    next(e);
+  }
+});
+
+router.post('/chart', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { aggFn, endTime, granularity, groupBy, name, q, startTime } =
+      req.body;
+
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    const startTimeNum = parseInt(startTime as string);
+    const endTimeNum = parseInt(endTime as string);
+    if (!isNumber(startTimeNum) || !isNumber(endTimeNum) || !name) {
+      return res.sendStatus(400);
+    }
+
+    // FIXME: separate name + dataType
+    const [metricName, metricDataType] = (name as string).split(' - ');
+    if (metricName == null || metricDataType == null) {
+      return res.sendStatus(400);
+    }
+
+    res.json(
+      await clickhouse.getMetricsChart({
+        aggFn: aggFn as clickhouse.AggFn,
+        dataType: metricDataType,
+        endTime: endTimeNum,
+        granularity,
+        groupBy: groupBy as string,
+        name: metricName,
+        q: q as string,
+        startTime: startTimeNum,
+        teamId: teamId.toString(),
+      }),
+    );
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+    next(e);
+  }
+});
+
+export default router;

packages/api/src/routers/api/root.ts

@@ -0,0 +1,117 @@
+import express from 'express';
+import isemail from 'isemail';
+import { serializeError } from 'serialize-error';
+
+import * as config from '../../config';
+import User from '../../models/user'; // TODO -> do not import model directly
+import logger from '../../utils/logger';
+import passport from '../../utils/passport';
+import { Api404Error } from '../../utils/errors';
+import { isTeamExisting, createTeam, getTeam } from '../../controllers/team';
+import { validatePassword } from '../../utils/validators';
+import {
+  isUserAuthenticated,
+  redirectToDashboard,
+  handleAuthError,
+} from '../../middleware/auth';
+
+const router = express.Router();
+
+router.get('/health', async (req, res) => {
+  res.send({ data: 'OK', version: config.CODE_VERSION, ip: req.ip });
+});
+
+router.get('/me', isUserAuthenticated, async (req, res, next) => {
+  try {
+    if (req.user == null) {
+      throw new Api404Error('Request without user found');
+    }
+
+    const { _id: id, team: teamId, email, name, createdAt } = req.user;
+
+    const team = await getTeam(teamId);
+
+    return res.json({
+      createdAt,
+      email,
+      id,
+      name,
+      team,
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.get('/installation', async (req, res, next) => {
+  try {
+    const _isTeamExisting = await isTeamExisting();
+    return res.json({
+      isTeamExisting: _isTeamExisting,
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.post(
+  '/login/password',
+  passport.authenticate('local', {
+    failWithError: true,
+    failureMessage: true,
+  }),
+  redirectToDashboard,
+  handleAuthError,
+);
+
+router.post('/register/password', async (req, res, next) => {
+  try {
+    const { email, password } = req.body;
+
+    if (!email || !password) {
+      return res.redirect(`${config.FRONTEND_URL}/register?err=missing`);
+    }
+
+    if (!isemail.validate(email) || !validatePassword(password)) {
+      return res.redirect(`${config.FRONTEND_URL}/register?err=invalid`);
+    }
+
+    if (await isTeamExisting()) {
+      return res.redirect(
+        `${config.FRONTEND_URL}/register?err=teamAlreadyExists`,
+      );
+    }
+
+    (User as any).register(
+      new User({ email }),
+      password,
+      async (err: Error, user: any) => {
+        if (err) {
+          logger.error(serializeError(err));
+          return res.redirect(`${config.FRONTEND_URL}/register?err=invalid`);
+        }
+
+        const team = await createTeam({
+          name: `${email}'s Team`,
+        });
+        user.team = team._id;
+        user.name = email;
+        await user.save();
+
+        return passport.authenticate('local')(req, res, () => {
+          redirectToDashboard(req, res);
+        });
+      },
+    );
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.get('/logout', (req, res) => {
+  // @ts-ignore
+  req.logout();
+  res.redirect(`${config.FRONTEND_URL}/login`);
+});
+
+export default router;

packages/api/src/routers/api/sessions.ts

@@ -0,0 +1,112 @@
+import express from 'express';
+import opentelemetry, { SpanStatusCode } from '@opentelemetry/api';
+import { isNumber, parseInt } from 'lodash';
+import { serializeError } from 'serialize-error';
+
+import * as clickhouse from '../../clickhouse';
+import { isUserAuthenticated } from '../../middleware/auth';
+import logger from '../../utils/logger';
+import { getTeam } from '../../controllers/team';
+
+const router = express.Router();
+
+router.get('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { endTime, q, startTime } = req.query;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    const startTimeNum = parseInt(startTime as string);
+    const endTimeNum = parseInt(endTime as string);
+    if (!isNumber(startTimeNum) || !isNumber(endTimeNum)) {
+      return res.sendStatus(400);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      return res.sendStatus(403);
+    }
+
+    res.json(
+      await clickhouse.getSessions({
+        endTime: endTimeNum,
+        limit: 500, // fixed limit for now
+        offset: 0, // fixed offset for now
+        q: q as string,
+        startTime: startTimeNum,
+        tableVersion: team.logStreamTableVersion,
+        teamId: teamId.toString(),
+      }),
+    );
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+
+    next(e);
+  }
+});
+
+router.get('/:sessionId/rrweb', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const { sessionId } = req.params;
+    const { endTime, limit, offset, startTime } = req.query;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    const startTimeNum = parseInt(startTime as string);
+    const endTimeNum = parseInt(endTime as string);
+    const limitNum = parseInt(limit as string);
+    const offsetNum = parseInt(offset as string);
+    if (
+      !isNumber(startTimeNum) ||
+      !isNumber(endTimeNum) ||
+      !isNumber(limitNum) ||
+      !isNumber(offsetNum)
+    ) {
+      return res.sendStatus(400);
+    }
+
+    const MAX_LIMIT = 1e6;
+
+    res.setHeader('Cache-Control', 'no-cache');
+    res.setHeader('Content-Type', 'text/event-stream');
+    res.setHeader('Connection', 'keep-alive');
+    res.flushHeaders(); // flush the headers to establish SSE with client
+
+    const stream = await clickhouse.getRrwebEvents({
+      sessionId: sessionId as string,
+      startTime: startTimeNum,
+      endTime: endTimeNum,
+      limit: Math.min(MAX_LIMIT, limitNum),
+      offset: offsetNum,
+    });
+
+    stream.on('data', (rows: any[]) => {
+      res.write(`${rows.map(row => `data: ${row.text}`).join('\n')}\n\n`);
+      res.flush();
+    });
+    stream.on('end', () => {
+      logger.info('Stream ended');
+
+      res.write('event: end\ndata:\n\n');
+      res.end();
+    });
+  } catch (e) {
+    const span = opentelemetry.trace.getActiveSpan();
+    span.recordException(e as Error);
+    span.setStatus({ code: SpanStatusCode.ERROR });
+    // WARNING: no need to call next(e) here, as the stream will be closed
+    logger.error({
+      message: 'Error while streaming rrweb events',
+      error: serializeError(e),
+      teamId: req.user?.team,
+      query: req.query,
+    });
+    res.end();
+  }
+});
+
+export default router;

packages/api/src/routers/api/team.ts

@@ -0,0 +1,187 @@
+import crypto from 'crypto';
+
+import express from 'express';
+import isemail from 'isemail';
+import pick from 'lodash/pick';
+import { serializeError } from 'serialize-error';
+
+import * as config from '../../config';
+import TeamInvite from '../../models/teamInvite';
+import User from '../../models/user';
+import logger from '../../utils/logger';
+import { findUserByEmail, findUsersByTeam } from '../../controllers/user';
+import { getTeam, rotateTeamApiKey } from '../../controllers/team';
+import {
+  isUserAuthenticated,
+  redirectToDashboard,
+} from '../../middleware/auth';
+import { validatePassword } from '../../utils/validators';
+
+const router = express.Router();
+
+router.post('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const { email: toEmail, name } = req.body;
+
+    if (!toEmail || !isemail.validate(toEmail)) {
+      return res.status(400).json({
+        message: 'Invalid email',
+      });
+    }
+
+    const teamId = req.user?.team;
+    const fromEmail = req.user?.email;
+
+    if (teamId == null) {
+      throw new Error(`User ${req.user?._id} not associated with a team`);
+    }
+
+    if (fromEmail == null) {
+      throw new Error(`User ${req.user?._id} doesnt have email`);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      throw new Error(`Team ${teamId} not found`);
+    }
+
+    const toUser = await findUserByEmail(toEmail);
+    if (toUser) {
+      return res.status(400).json({
+        message: 'User already exists. Please contact HyperDX team for support',
+      });
+    }
+
+    let teamInvite = await TeamInvite.findOne({
+      teamId: team._id,
+      email: toEmail, // TODO: case insensitive ?
+    });
+
+    if (!teamInvite) {
+      teamInvite = await new TeamInvite({
+        teamId: team._id,
+        name,
+        email: toEmail, // TODO: case insensitive ?
+        token: crypto.randomBytes(32).toString('hex'),
+      }).save();
+    }
+
+    res.json({
+      url: `${config.FRONTEND_URL}/join-team?token=${teamInvite.token}`,
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.get('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    const userId = req.user?._id;
+
+    if (teamId == null) {
+      throw new Error(`User ${req.user?._id} not associated with a team`);
+    }
+    if (userId == null) {
+      throw new Error(`User has no id`);
+    }
+
+    const team = await getTeam(teamId);
+    if (team == null) {
+      throw new Error(`Team ${teamId} not found for user ${userId}`);
+    }
+
+    const teamUsers = await findUsersByTeam(teamId);
+    const teamInvites = await TeamInvite.find({});
+
+    res.json({
+      ...pick(team.toJSON(), [
+        '_id',
+        'allowedAuthMethods',
+        'apiKey',
+        'archive',
+        'name',
+        'slackAlert',
+      ]),
+      users: teamUsers.map(user => ({
+        ...pick(user.toJSON({ virtuals: true }), [
+          'email',
+          'name',
+          'hasPasswordAuth',
+        ]),
+        isCurrentUser: user._id.equals(userId),
+      })),
+      teamInvites: teamInvites.map(ti => ({
+        createdAt: ti.createdAt,
+        email: ti.email,
+        name: ti.name,
+        url: `${config.FRONTEND_URL}/join-team?token=${ti.token}`,
+      })),
+    });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.patch('/apiKey', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    if (teamId == null) {
+      throw new Error(`User ${req.user?._id} not associated with a team`);
+    }
+    const team = await rotateTeamApiKey(teamId);
+    res.json({ newApiKey: team?.apiKey });
+  } catch (e) {
+    next(e);
+  }
+});
+
+router.post('/setup/:token', async (req, res, next) => {
+  try {
+    const { password } = req.body;
+    const { token } = req.params;
+
+    if (!validatePassword(password)) {
+      return res.redirect(
+        `${config.FRONTEND_URL}/join-team?err=invalid&token=${token}`,
+      );
+    }
+
+    const teamInvite = await TeamInvite.findOne({
+      token: req.params.token,
+    });
+    if (!teamInvite) {
+      return res.status(401).send('Invalid token');
+    }
+
+    (User as any).register(
+      new User({
+        email: teamInvite.email,
+        name: teamInvite.email,
+        team: teamInvite.teamId,
+      }),
+      password, // TODO: validate password
+      async (err: Error, user: any) => {
+        if (err) {
+          logger.error(serializeError(err));
+          return res.redirect(
+            `${config.FRONTEND_URL}/join-team?token=${token}&err=500`,
+          );
+        }
+
+        await TeamInvite.findByIdAndRemove(teamInvite._id);
+
+        req.login(user, err => {
+          if (err) {
+            return next(err);
+          }
+          redirectToDashboard(req, res);
+        });
+      },
+    );
+  } catch (e) {
+    next(e);
+  }
+});
+
+export default router;

packages/api/src/routers/api/webhooks.ts

@@ -0,0 +1,72 @@
+import express from 'express';
+
+import Webhook from '../../models/webhook';
+import { isUserAuthenticated } from '../../middleware/auth';
+
+const router = express.Router();
+
+router.get('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    const { service } = req.query;
+    const webhooks = await Webhook.find(
+      { team: teamId, service },
+      { __v: 0, team: 0 },
+    );
+    res.json({
+      data: webhooks,
+    });
+  } catch (err) {
+    next(err);
+  }
+});
+
+router.post('/', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    const { name, service, url } = req.body;
+    if (!service || !url || !name) return res.sendStatus(400);
+    const totalWebhooks = await Webhook.countDocuments({
+      team: teamId,
+      service,
+    });
+    if (totalWebhooks >= 5) {
+      return res.status(400).json({
+        message: 'You can only have 5 webhooks per team per service',
+      });
+    }
+    if (await Webhook.findOne({ team: teamId, service, url })) {
+      return res.status(400).json({
+        message: 'Webhook already exists',
+      });
+    }
+    const webhook = new Webhook({ team: teamId, service, url, name });
+    await webhook.save();
+    res.json({
+      data: webhook,
+    });
+  } catch (err) {
+    next(err);
+  }
+});
+
+router.delete('/:id', isUserAuthenticated, async (req, res, next) => {
+  try {
+    const teamId = req.user?.team;
+    if (teamId == null) {
+      return res.sendStatus(403);
+    }
+    await Webhook.findOneAndDelete({ _id: req.params.id, team: teamId });
+    res.json({});
+  } catch (err) {
+    next(err);
+  }
+});
+
+export default router;

packages/api/src/server.ts

@@ -0,0 +1,75 @@
+import http from 'http';
+
+import { serializeError } from 'serialize-error';
+
+import * as clickhouse from './clickhouse';
+import * as config from './config';
+import logger from './utils/logger';
+import redisClient from './utils/redis';
+import { connectDB, mongooseConnection } from './models';
+
+export default class Server {
+  protected httpServer: http.Server;
+
+  private async createServer() {
+    switch (config.APP_TYPE) {
+      case 'api':
+        return http.createServer(
+          (await import('./api-app').then(m => m.default)) as any,
+        );
+      case 'aggregator':
+        return http.createServer(
+          (await import('./aggregator-app').then(m => m.default)) as any,
+        );
+      default:
+        throw new Error(`Invalid APP_TYPE: ${config.APP_TYPE}`);
+    }
+  }
+
+  async start() {
+    this.httpServer = await this.createServer();
+    this.httpServer.keepAliveTimeout = 61000; // Ensure all inactive connections are terminated by the ALB, by setting this a few seconds higher than the ALB idle timeout
+    this.httpServer.headersTimeout = 62000; // Ensure the headersTimeout is set higher than the keepAliveTimeout due to this nodejs regression bug: https://github.com/nodejs/node/issues/27363
+
+    this.httpServer.listen(config.PORT, () => {
+      logger.info(
+        `Server listening on port ${config.PORT}, NODE_ENV=${process.env.NODE_ENV}`,
+      );
+    });
+
+    await Promise.all([
+      connectDB(),
+      redisClient.connect(),
+      clickhouse.connect(),
+    ]);
+  }
+
+  // graceful shutdown
+  stop() {
+    this.httpServer.close(closeServerErr => {
+      if (closeServerErr) {
+        logger.error(serializeError(closeServerErr));
+      }
+      logger.info('Http server closed.');
+      redisClient
+        .disconnect()
+        .then(() => {
+          logger.info('Redis client disconnected.');
+        })
+        .catch((err: any) => {
+          logger.error(serializeError(err));
+        });
+      mongooseConnection.close(false, closeDBConnectionErr => {
+        if (closeDBConnectionErr) {
+          logger.error(serializeError(closeDBConnectionErr));
+        }
+        logger.info('Mongo connection closed.');
+
+        if (closeServerErr || closeDBConnectionErr) {
+          process.exit(1);
+        }
+        process.exit(0);
+      });
+    });
+  }
+}

packages/api/src/tasks/__tests__/checkAlerts.test.ts

@@ -0,0 +1,52 @@
+import { buildLogSearchLink, roundDownToXMinutes } from '../checkAlerts';
+
+describe('checkAlerts', () => {
+  it('roundDownToXMinutes', () => {
+    // 1 min
+    const roundDownTo1Minute = roundDownToXMinutes(1);
+    expect(
+      roundDownTo1Minute(new Date('2023-03-17T22:13:03.103Z')).toISOString(),
+    ).toBe('2023-03-17T22:13:00.000Z');
+    expect(
+      roundDownTo1Minute(new Date('2023-03-17T22:13:59.103Z')).toISOString(),
+    ).toBe('2023-03-17T22:13:00.000Z');
+
+    // 5 mins
+    const roundDownTo5Minutes = roundDownToXMinutes(5);
+    expect(
+      roundDownTo5Minutes(new Date('2023-03-17T22:13:03.103Z')).toISOString(),
+    ).toBe('2023-03-17T22:10:00.000Z');
+    expect(
+      roundDownTo5Minutes(new Date('2023-03-17T22:17:59.103Z')).toISOString(),
+    ).toBe('2023-03-17T22:15:00.000Z');
+    expect(
+      roundDownTo5Minutes(new Date('2023-03-17T22:59:59.103Z')).toISOString(),
+    ).toBe('2023-03-17T22:55:00.000Z');
+  });
+
+  it('buildLogSearchLink', () => {
+    expect(
+      buildLogSearchLink({
+        startTime: new Date('2023-03-17T22:13:03.103Z'),
+        endTime: new Date('2023-03-17T22:13:59.103Z'),
+        logView: {
+          _id: 123,
+        } as any,
+      }),
+    ).toBe(
+      'http://localhost:9090/search/123?from=1679091183103&to=1679091239103',
+    );
+    expect(
+      buildLogSearchLink({
+        startTime: new Date('2023-03-17T22:13:03.103Z'),
+        endTime: new Date('2023-03-17T22:13:59.103Z'),
+        logView: {
+          _id: 123,
+        } as any,
+        q: '🐱 foo:"bar"',
+      }),
+    ).toBe(
+      'http://localhost:9090/search/123?from=1679091183103&to=1679091239103&q=%F0%9F%90%B1+foo%3A%22bar%22',
+    );
+  });
+});

packages/api/src/tasks/checkAlerts.ts

@@ -0,0 +1,297 @@
+// --------------------------------------------------------
+// -------------- EXECUTE EVERY MINUTE --------------------
+// --------------------------------------------------------
+import { URLSearchParams } from 'url';
+
+import * as fns from 'date-fns';
+import * as fnsTz from 'date-fns-tz';
+import ms from 'ms';
+import { serializeError } from 'serialize-error';
+
+import * as clickhouse from '../clickhouse';
+import * as config from '../config';
+import * as slack from '../utils/slack';
+import Alert, { AlertState, IAlert } from '../models/alert';
+import AlertHistory, { IAlertHistory } from '../models/alertHistory';
+import LogView from '../models/logView';
+import Webhook from '../models/webhook';
+import logger from '../utils/logger';
+import { ITeam } from '../models/team';
+import { ObjectId } from '../models';
+import { truncateString } from '../utils/common';
+
+import type { ResponseJSON } from '@clickhouse/client';
+import type { LogSearchRow } from '../clickhouse';
+
+const MAX_MESSAGE_LENGTH = 500;
+
+const getLogViewEnhanced = async (logViewId: ObjectId) => {
+  const logView = await LogView.findById(logViewId).populate<{
+    team: ITeam;
+  }>('team');
+  if (!logView) {
+    throw new Error(`LogView ${logViewId} not found `);
+  }
+  return logView;
+};
+
+export const buildLogSearchLink = ({
+  endTime,
+  logView,
+  q,
+  startTime,
+}: {
+  endTime: Date;
+  logView: Awaited<ReturnType<typeof getLogViewEnhanced>>;
+  q?: string;
+  startTime: Date;
+}) => {
+  const url = new URL(`${config.FRONTEND_URL}/search/${logView._id}`);
+  const queryParams = new URLSearchParams({
+    from: startTime.getTime().toString(),
+    to: endTime.getTime().toString(),
+  });
+  if (q) {
+    queryParams.append('q', q);
+  }
+  url.search = queryParams.toString();
+  return url.toString();
+};
+
+const buildEventSlackMessage = ({
+  alert,
+  endTime,
+  group,
+  logView,
+  results,
+  searchQuery,
+  startTime,
+  totalCount,
+}: {
+  alert: IAlert;
+  endTime: Date;
+  group?: string;
+  logView: Awaited<ReturnType<typeof getLogViewEnhanced>>;
+  results: ResponseJSON<LogSearchRow> | undefined;
+  searchQuery?: string;
+  startTime: Date;
+  totalCount: number;
+}) => {
+  const mrkdwn = [
+    `*<${buildLogSearchLink({
+      endTime,
+      logView,
+      q: searchQuery,
+      startTime,
+    })} | Alert for ${logView.name}>*`,
+    ...(group != null ? [`Group: "${group}"`] : []),
+    `${totalCount} lines found, expected ${
+      alert.type === 'presence' ? 'less than' : 'greater than'
+    } ${alert.threshold} lines`,
+    ...(results?.rows != null && totalCount > 0
+      ? [
+          `\`\`\``,
+          truncateString(
+            results.data
+              .map(row => {
+                return `${fnsTz.formatInTimeZone(
+                  new Date(row.timestamp),
+                  'Etc/UTC',
+                  'MMM d HH:mm:ss',
+                )}Z [${row.severity_text}] ${truncateString(
+                  row.body,
+                  MAX_MESSAGE_LENGTH,
+                )}`;
+              })
+              .join('\n'),
+            2500,
+          ),
+          `\`\`\``,
+        ]
+      : []),
+  ].join('\n');
+
+  return {
+    text: `Alert for ${logView.name} - ${totalCount} lines found`,
+    blocks: [
+      {
+        type: 'section',
+        text: {
+          type: 'mrkdwn',
+          text: mrkdwn,
+        },
+      },
+    ],
+  };
+};
+
+const fireChannelEvent = async ({
+  alert,
+  logView,
+  totalCount,
+  group,
+  startTime,
+  endTime,
+}: {
+  alert: IAlert;
+  logView: Awaited<ReturnType<typeof getLogViewEnhanced>>;
+  totalCount: number;
+  group?: string;
+  startTime: Date;
+  endTime: Date;
+}) => {
+  const searchQuery = alert.groupBy
+    ? `${logView.query} ${alert.groupBy}:"${group}"`
+    : logView.query;
+  // TODO: show group + total count for group-by alerts
+  const results = await clickhouse.getLogBatch({
+    endTime: endTime.getTime(),
+    limit: 5,
+    offset: 0,
+    order: 'desc', // TODO: better to use null
+    q: searchQuery,
+    startTime: startTime.getTime(),
+    tableVersion: logView.team.logStreamTableVersion,
+    teamId: logView.team._id.toString(),
+  });
+
+  switch (alert.channel.type) {
+    case 'webhook': {
+      const webhook = await Webhook.findOne({
+        _id: alert.channel.webhookId,
+      });
+      // ONLY SUPPORTS SLACK WEBHOOKS FOR NOW
+      if (webhook.service === 'slack') {
+        await slack.postMessageToWebhook(
+          webhook.url,
+          buildEventSlackMessage({
+            alert,
+            endTime,
+            group,
+            logView,
+            results,
+            searchQuery,
+            startTime,
+            totalCount,
+          }),
+        );
+      }
+      break;
+    }
+    default:
+      throw new Error(
+        `Unsupported channel type: ${(alert.channel as any).any}`,
+      );
+  }
+};
+
+const doesExceedThreshold = (alert: IAlert, totalCount: number) => {
+  if (alert.type === 'presence' && totalCount >= alert.threshold) {
+    return true;
+  } else if (alert.type === 'absence' && totalCount < alert.threshold) {
+    return true;
+  }
+  return false;
+};
+
+export const roundDownTo = (roundTo: number) => (x: Date) =>
+  new Date(Math.floor(x.getTime() / roundTo) * roundTo);
+export const roundDownToXMinutes = (x: number) => roundDownTo(1000 * 60 * x);
+
+const processAlert = async (now: Date, alert: IAlert) => {
+  try {
+    const logView = await getLogViewEnhanced(alert.logView);
+
+    const previous: IAlertHistory | undefined = (
+      await AlertHistory.find({ alert: alert._id })
+        .sort({ createdAt: -1 })
+        .limit(1)
+    )[0];
+
+    const windowSizeInMins = ms(alert.interval) / 60000;
+    const nowInMinsRoundDown = roundDownToXMinutes(windowSizeInMins)(now);
+    if (
+      previous &&
+      fns.getTime(previous.createdAt) === fns.getTime(nowInMinsRoundDown)
+    ) {
+      logger.info({
+        message: `Skipped to check alert since the time diff is still less than 1 window size`,
+        windowSizeInMins,
+        nowInMinsRoundDown,
+        previous,
+        now,
+        alert,
+        logView,
+      });
+      return;
+    }
+    const history = await new AlertHistory({
+      alert: alert._id,
+      createdAt: nowInMinsRoundDown,
+    }).save();
+    const checkStartTime = previous
+      ? previous.createdAt
+      : fns.subMinutes(nowInMinsRoundDown, windowSizeInMins);
+    const checkEndTime = nowInMinsRoundDown;
+    const check = await clickhouse.checkAlert({
+      endTime: checkEndTime,
+      groupBy: alert.groupBy,
+      q: logView.query,
+      startTime: checkStartTime,
+      tableVersion: logView.team.logStreamTableVersion,
+      teamId: logView.team._id.toString(),
+      windowSizeInMins,
+    });
+
+    logger.info({
+      message: 'Received alert metric',
+      alert,
+      logView,
+      check,
+      checkStartTime,
+      checkEndTime,
+    });
+
+    // TODO: support INSUFFICIENT_DATA state
+    let alertState = AlertState.OK;
+    if (check?.rows && check?.rows > 0) {
+      for (const checkData of check.data) {
+        const totalCount = parseInt(checkData.count);
+        if (doesExceedThreshold(alert, totalCount)) {
+          alertState = AlertState.ALERT;
+          logger.info({
+            message: `Triggering ${alert.channel.type} alarm!`,
+            alert,
+            logView,
+            totalCount,
+            checkData,
+          });
+          const bucketStart = new Date(checkData.ts_bucket);
+          await fireChannelEvent({
+            alert,
+            logView,
+            totalCount,
+            group: checkData.group,
+            startTime: bucketStart,
+            endTime: fns.addMinutes(bucketStart, windowSizeInMins),
+          });
+          history.counts += 1;
+        }
+      }
+      await history.save();
+    }
+    alert.state = alertState;
+    await (alert as any).save();
+  } catch (e) {
+    // Uncomment this for better error messages locally
+    // console.error(e);
+    logger.error(serializeError(e));
+  }
+};
+
+export default async () => {
+  const now = new Date();
+  const alerts = await Alert.find({});
+  logger.info(`Going to process ${alerts.length} alerts`);
+  await Promise.all(alerts.map(alert => processAlert(now, alert)));
+};

packages/api/src/tasks/index.ts

@@ -0,0 +1,84 @@
+import { performance } from 'perf_hooks';
+
+import minimist from 'minimist';
+import schedule from 'node-schedule';
+import { serializeError } from 'serialize-error';
+
+import checkAlerts from './checkAlerts';
+import logger from '../utils/logger';
+import redisClient from '../utils/redis';
+import refreshPropertyTypeMappings from './refreshPropertyTypeMappings';
+import { IS_DEV } from '../config';
+import { connectDB, mongooseConnection } from '../models';
+
+const main = async () => {
+  const argv = minimist(process.argv.slice(2));
+  const taskName = argv._[0];
+
+  // connect dbs + redis
+  await Promise.all([connectDB(), redisClient.connect()]);
+
+  const t0 = performance.now();
+  logger.info(`Task [${taskName}] started at ${new Date()}`);
+  switch (taskName) {
+    case 'check-alerts':
+      await checkAlerts();
+      break;
+    case 'refresh-property-type-mappings':
+      await refreshPropertyTypeMappings();
+      break;
+    // only for testing
+    case 'ping-pong':
+      logger.info(`
+                 O .
+               _/|\\_-O
+              ___|_______
+             /     |     \
+            /      |      \
+           #################
+          /   _ ( )|        \
+         /    ( ) ||         \
+        /  \\  |_/ |          \
+       /____\\/|___|___________\
+          |    |             |
+          |   / \\           |
+          |  /   \\          |
+          |_/    /_
+      `);
+      break;
+    default:
+      throw new Error(`Unkown task name ${taskName}`);
+  }
+  logger.info(
+    `Task [${taskName}] finished in ${(performance.now() - t0).toFixed(2)} ms`,
+  );
+
+  // close redis + db connections
+  await Promise.all([redisClient.disconnect(), mongooseConnection.close()]);
+};
+
+if (IS_DEV) {
+  schedule.scheduleJob('*/1 * * * *', main);
+} else {
+  main()
+    .then(() => {
+      process.exit(0);
+    })
+    .catch(err => {
+      console.log(err);
+      logger.error(serializeError(err));
+      process.exit(1);
+    });
+}
+
+process.on('uncaughtException', (err: Error) => {
+  console.log(err);
+  logger.error(serializeError(err));
+  process.exit(1);
+});
+
+process.on('unhandledRejection', (err: Error) => {
+  console.log(err);
+  logger.error(serializeError(err));
+  process.exit(1);
+});

packages/api/src/tasks/refreshPropertyTypeMappings.ts

@@ -0,0 +1,42 @@
+// --------------------------------------------------------
+// ------------------ EXECUTE HOURLY ----------------------
+// --------------------------------------------------------
+import ms from 'ms';
+
+import * as clickhouse from '../clickhouse';
+import Team from '../models/team';
+import logger from '../utils/logger';
+import { LogsPropertyTypeMappingsModel } from '../clickhouse/propertyTypeMappingsModel';
+
+const MAX_PROCESS_TEAMS = 30;
+const LOG_PREFIX = '[refreshPropertyTypeMappings]';
+
+export default async () => {
+  const nowInMs = Date.now();
+  const teams = await Team.find({});
+  let c = 0;
+  const promises = [];
+  for (const team of teams) {
+    if (c >= MAX_PROCESS_TEAMS) {
+      logger.info(`${LOG_PREFIX} Processed ${c} teams, exiting...`);
+      break;
+    }
+    const teamId = team._id.toString();
+    const model = new LogsPropertyTypeMappingsModel(
+      team.logStreamTableVersion,
+      teamId,
+      clickhouse.fetchLogsPropertyTypeMappings(nowInMs - ms('3d'), nowInMs),
+    );
+    const isAboutToExpire = await model.isAboutToExpire();
+    if (isAboutToExpire) {
+      logger.info(`${LOG_PREFIX} Refreshing team ${teamId}`);
+      promises.push(model._refresh({ incrementalUpdate: false }));
+      c += 1;
+    } else {
+      logger.info(`${LOG_PREFIX} Skipping team ${teamId}`);
+    }
+  }
+
+  await Promise.all(promises);
+  logger.info(`${LOG_PREFIX} Refreshed ${c} teams`);
+};

packages/api/src/utils/__tests__/errors.test.ts

@@ -0,0 +1,21 @@
+import { Api500Error, BaseError, isOperationalError } from '../errors';
+
+describe('Errors utils', () => {
+  test('BaseError class', () => {
+    const e = new BaseError('nvim', 500, true, 'is the best editor!!!');
+    expect(e.name).toBe('nvim');
+    expect(e.statusCode).toBe(500);
+    expect(e.isOperational).toBeTruthy();
+    expect(e.message).toBe('is the best editor!!!');
+    expect(e.stack?.includes('nvim: is the best editor'));
+  });
+
+  test('isOperational', () => {
+    expect(
+      isOperationalError(
+        new BaseError('nvim', 500, true, 'is the best editor!!!'),
+      ),
+    ).toBeTruthy();
+    expect(isOperationalError(new Api500Error('BANG'))).toBeTruthy();
+  });
+});

packages/api/src/utils/__tests__/logParser.test.ts

@@ -0,0 +1,121 @@
+import { mapObjectToKeyValuePairs, traverseJson } from '../logParser';
+
+describe('logParser', () => {
+  it('traverseJson', () => {
+    const jsonIt = traverseJson({
+      foo: {
+        bar: 'bar',
+        foo1: {
+          foo1: 'bar1',
+          foo2: {
+            bar2: 'bar2',
+          },
+        },
+      },
+    });
+    const keys = [];
+    const values = [];
+    for (const [key, value] of jsonIt) {
+      keys.push(key);
+      values.push(value);
+    }
+
+    expect(keys).toEqual([
+      ['foo'],
+      ['foo', 'bar'],
+      ['foo', 'foo1'],
+      ['foo', 'foo1', 'foo1'],
+      ['foo', 'foo1', 'foo2'],
+      ['foo', 'foo1', 'foo2', 'bar2'],
+    ]);
+    expect(values).toEqual([
+      {
+        bar: 'bar',
+        foo1: {
+          foo1: 'bar1',
+          foo2: {
+            bar2: 'bar2',
+          },
+        },
+      },
+      'bar',
+      {
+        foo1: 'bar1',
+        foo2: {
+          bar2: 'bar2',
+        },
+      },
+      'bar1',
+      {
+        bar2: 'bar2',
+      },
+      'bar2',
+    ]);
+  });
+
+  it('mapObjectToKeyValuePairs', () => {
+    expect(mapObjectToKeyValuePairs(null as any)).toEqual({
+      'bool.names': [],
+      'bool.values': [],
+      'number.names': [],
+      'number.values': [],
+      'string.names': [],
+      'string.values': [],
+    });
+
+    expect(mapObjectToKeyValuePairs({})).toEqual({
+      'bool.names': [],
+      'bool.values': [],
+      'number.names': [],
+      'number.values': [],
+      'string.names': [],
+      'string.values': [],
+    });
+
+    expect(
+      mapObjectToKeyValuePairs({ foo: '123', foo1: 123, foo2: false }),
+    ).toEqual({
+      'bool.names': ['foo2'],
+      'bool.values': [0],
+      'number.names': ['foo1'],
+      'number.values': [123],
+      'string.names': ['foo'],
+      'string.values': ['123'],
+    });
+
+    expect(
+      mapObjectToKeyValuePairs({
+        foo: '123',
+        foo1: 123,
+        foo2: false,
+        nested: { foo: 'bar' },
+        good: {
+          burrito: {
+            is: true,
+          },
+        },
+        array1: [456],
+        array2: [
+          'foo1',
+          {
+            foo2: 'bar2',
+          },
+          [
+            {
+              foo3: 'bar3',
+            },
+          ],
+        ],
+      }),
+    ).toMatchSnapshot();
+
+    const testObject = {};
+    for (let i = 0; i < 2000; i++) {
+      testObject[`foo${i}`] = i;
+    }
+    const result = mapObjectToKeyValuePairs(testObject);
+    expect(result['number.names'].length).toEqual(1024);
+    expect(result['number.values'].length).toEqual(1024);
+    expect(result).toMatchSnapshot();
+  });
+});

packages/api/src/utils/__tests__/validators.test.ts

@@ -0,0 +1,17 @@
+import * as validators from '../validators';
+
+describe('validators', () => {
+  describe('validatePassword', () => {
+    it('should return true if password is valid', () => {
+      expect(validators.validatePassword('abcdefgh')).toBe(true);
+    });
+
+    it('should return false if password is invalid', () => {
+      expect(validators.validatePassword(null)).toBe(false);
+      expect(validators.validatePassword(undefined)).toBe(false);
+      expect(validators.validatePassword('')).toBe(false);
+      expect(validators.validatePassword('1234567')).toBe(false);
+      expect(validators.validatePassword('a'.repeat(65))).toBe(false);
+    });
+  });
+});

packages/api/src/utils/common.ts

@@ -0,0 +1,28 @@
+export const useTry = <T>(fn: () => T): [null | Error | unknown, null | T] => {
+  let output = null;
+  let error = null;
+  try {
+    output = fn();
+    return [error, output];
+  } catch (e) {
+    error = e;
+    return [error, output];
+  }
+};
+
+export const tryJSONStringify = (
+  json: Record<string, unknown> | Record<string, unknown>[],
+) => {
+  const [_, result] = useTry<string>(() => JSON.stringify(json));
+  return result;
+};
+
+export const truncateString = (str: string, length: number) => {
+  if (str.length > length) {
+    return str.substring(0, length) + '...';
+  }
+  return str;
+};
+
+export const sleep = (ms: number) =>
+  new Promise(resolve => setTimeout(resolve, ms));

packages/api/src/utils/email.ts

@@ -0,0 +1,27 @@
+export const sendAlert = ({
+  alertDetails,
+  alertEvents,
+  alertGroup,
+  alertName,
+  alertUrl,
+  toEmail,
+}: {
+  alertDetails: string;
+  alertEvents: string;
+  alertGroup?: string;
+  alertName: string;
+  alertUrl: string;
+  toEmail: string;
+}) => {
+  // Send alert email
+};
+
+export const sendResetPasswordEmail = ({
+  toEmail,
+  token,
+}: {
+  toEmail: string;
+  token: string;
+}) => {
+  // Send reset password email
+};

packages/api/src/utils/errors.ts

@@ -0,0 +1,56 @@
+export enum StatusCode {
+  OK = 200,
+  BAD_REQUEST = 400,
+  UNAUTHORIZED = 401,
+  FORBIDDEN = 403,
+  NOT_FOUND = 404,
+  INTERNAL_SERVER = 500,
+}
+
+export class BaseError extends Error {
+  name: string;
+
+  statusCode: StatusCode;
+
+  isOperational: boolean;
+
+  constructor(
+    name: string,
+    statusCode: StatusCode,
+    isOperational: boolean,
+    description: string,
+  ) {
+    super(description);
+
+    Object.setPrototypeOf(this, BaseError.prototype);
+
+    this.name = name;
+    this.statusCode = statusCode;
+    this.isOperational = isOperational;
+  }
+}
+
+export class Api500Error extends BaseError {
+  constructor(name: string) {
+    super(name, StatusCode.INTERNAL_SERVER, true, 'Internal Server Error');
+  }
+}
+
+export class Api400Error extends BaseError {
+  constructor(name: string) {
+    super(name, StatusCode.BAD_REQUEST, true, 'Bad Request');
+  }
+}
+
+export class Api404Error extends BaseError {
+  constructor(name: string) {
+    super(name, StatusCode.NOT_FOUND, true, 'Not Found');
+  }
+}
+
+export const isOperationalError = (error: Error) => {
+  if (error instanceof BaseError) {
+    return error.isOperational;
+  }
+  return false;
+};

packages/api/src/utils/logParser.ts

@@ -0,0 +1,307 @@
+import _ from 'lodash';
+
+import { tryJSONStringify } from './common';
+
+export type JSONBlob = Record<string, any>;
+
+export type KeyPath = string[];
+
+export enum LogType {
+  Log = 'log',
+  Metric = 'metric',
+  Span = 'span',
+}
+
+export enum LogPlatform {
+  Docker = 'docker',
+  Heroku = 'heroku',
+  HyperDX = 'hyperdx',
+  NodeJS = 'nodejs',
+  OtelLogs = 'otel-logs',
+  OtelTraces = 'otel-traces',
+  OtelMetrics = 'otel-metrics',
+  Rrweb = 'rrweb',
+}
+
+export type KeyValuePairs = {
+  'bool.names': string[];
+  'bool.values': number[];
+  'number.names': string[];
+  'number.values': number[];
+  'string.names': string[];
+  'string.values': string[];
+};
+
+export type SpanFields = {
+  end_timestamp?: number;
+  span_name?: string;
+  parent_span_id?: string;
+};
+
+export type LogFields = {
+  _host?: string;
+  severity_number?: number;
+  severity_text?: string;
+};
+
+export type RrwebEventModel = KeyValuePairs & {
+  _service?: string;
+  _source: string;
+  timestamp: number;
+};
+
+export type LogStreamModel = KeyValuePairs &
+  LogFields &
+  SpanFields & {
+    _namespace?: string;
+    _platform: LogPlatform;
+    _service?: string;
+    _source: string; // raw log
+    observed_timestamp: number;
+    span_id?: string;
+    timestamp: number;
+    trace_id?: string;
+    type: LogType;
+  };
+
+export type MetricModel = {
+  _string_attributes: Record<string, string>;
+  data_type: string;
+  name: string;
+  timestamp: number;
+  value: number;
+};
+
+const MAX_DEPTH = 6;
+export function* traverseJson(
+  currentNode: JSONBlob,
+  depth = 1,
+  keyPathArray?: KeyPath,
+): IterableIterator<[KeyPath, any]> {
+  for (const [key, value] of Object.entries(currentNode)) {
+    const keyPath = keyPathArray ? [...keyPathArray, key] : [key];
+
+    yield [keyPath, value];
+
+    if (_.isObject(value) && Object.keys(value).length && depth < MAX_DEPTH) {
+      // TODO: limit array length ??
+      if (_.isArray(value)) {
+        yield [keyPath, tryJSONStringify(value)];
+      } else {
+        yield* traverseJson(value, depth + 1, keyPath);
+      }
+    }
+    // TODO: alert if MAX_DEPTH is reached
+  }
+}
+
+const MAX_KEY_VALUE_PAIRS_LENGTH = 1024;
+export const mapObjectToKeyValuePairs = (
+  blob: JSONBlob,
+  maxArrayLength = MAX_KEY_VALUE_PAIRS_LENGTH,
+): KeyValuePairs => {
+  const output: KeyValuePairs = {
+    'bool.names': [],
+    'bool.values': [],
+    'number.names': [],
+    'number.values': [],
+    'string.names': [],
+    'string.values': [],
+  };
+  const pushArray = (
+    type: 'bool' | 'number' | 'string',
+    keyPath: string,
+    value: any,
+  ) => {
+    const keyNames = `${type}.names`;
+    const keyValues = `${type}.values`;
+    if (output[keyNames].length < maxArrayLength) {
+      output[keyNames].push(keyPath);
+      output[keyValues].push(value);
+      return true;
+    }
+    return false;
+  };
+
+  let reachedBoolMaxLength = false;
+  let reachedNumberMaxLength = false;
+  let reachedStringMaxLength = false;
+
+  if (_.isPlainObject(blob)) {
+    const jsonIt = traverseJson(blob);
+    let pushed = true;
+    for (const [key, value] of jsonIt) {
+      const compoundKeyPath = key.join('.');
+      if (!reachedNumberMaxLength && _.isNumber(value)) {
+        pushed = pushArray('number', compoundKeyPath, value);
+        if (!pushed) {
+          reachedNumberMaxLength = true;
+        }
+      } else if (!reachedBoolMaxLength && _.isBoolean(value)) {
+        pushed = pushArray('bool', compoundKeyPath, value ? 1 : 0);
+        if (!pushed) {
+          reachedBoolMaxLength = true;
+        }
+      } else if (!reachedStringMaxLength && _.isString(value)) {
+        pushed = pushArray('string', compoundKeyPath, value);
+        if (!pushed) {
+          reachedStringMaxLength = true;
+        }
+      }
+
+      if (
+        reachedBoolMaxLength &&
+        reachedNumberMaxLength &&
+        reachedStringMaxLength
+      ) {
+        console.warn(
+          `Max array length reached for ${compoundKeyPath} with value ${value}`,
+        );
+        break;
+      }
+    }
+  }
+
+  return output;
+};
+
+type _VectorLogFields = {
+  h?: string; // host
+  sn?: number; // severity number
+  st?: string; // severity text
+};
+
+type _VectorSpanFileds = {
+  et?: number; // end timestamp
+  p_id?: string; // parent id
+  s_n?: string; // span name
+};
+
+export type VectorLog = _VectorLogFields &
+  _VectorSpanFileds & {
+    authorization: string | null;
+    b: JSONBlob;
+    hdx_platform: LogPlatform;
+    hdx_token: string | null;
+    r: string; // raw
+    s_id?: string; // span id
+    sv: string; // service
+    t_id?: string; // trace id
+    ts: number; // timestamp
+    tso: number; // observed timestamp
+  };
+
+export type VectorSpan = {
+  atrs: JSONBlob; // attributes
+  authorization?: string | null;
+  et: number; // end timestamp
+  hdx_platform: string;
+  hdx_token: string | null;
+  n: string; // name
+  p_id: string; // parent id
+  r: string; // raw
+  s_id: string; // span id
+  st: number; // start timestamp
+  t_id: string; // trace id
+  tso: number; // observed timestamp
+};
+
+export type VectorMetric = {
+  authorization?: string;
+  b: JSONBlob; // tags
+  dt: string; // data type
+  hdx_platform: string;
+  hdx_token: string;
+  n: string; // name
+  ts: number; // timestamp
+  tso: number; // observed timestamp
+  v: number; // value
+};
+
+abstract class ParsingInterface<T> {
+  abstract _parse(
+    log: T,
+    ...args: any[]
+  ): LogStreamModel | MetricModel | RrwebEventModel;
+
+  parse(logs: T[], ...args: any[]) {
+    const parsedLogs = [];
+    for (const log of logs) {
+      try {
+        parsedLogs.push(this._parse(log, ...args));
+      } catch (e) {
+        // continue if parser fails to parse single log
+        console.warn(e);
+      }
+    }
+    return parsedLogs;
+  }
+}
+
+class VectorLogParser extends ParsingInterface<VectorLog> {
+  getType(log: VectorLog): LogType {
+    if (log.hdx_platform === LogPlatform.OtelTraces) {
+      return LogType.Span;
+    } else if (log.hdx_platform === LogPlatform.OtelMetrics) {
+      return LogType.Metric;
+    }
+    return LogType.Log;
+  }
+
+  _parse(log: VectorLog): LogStreamModel {
+    return {
+      ...mapObjectToKeyValuePairs(log.b),
+      _platform: log.hdx_platform,
+      _service: log.sv,
+      _source: log.r,
+      observed_timestamp: log.tso,
+      timestamp: log.ts,
+      type: this.getType(log),
+      // Log
+      _host: log.h,
+      severity_text: log.st,
+      severity_number: log.sn,
+      // Span
+      end_timestamp: log.et,
+      span_name: log.s_n,
+      parent_span_id: log.p_id,
+      span_id: log.s_id,
+      trace_id: log.t_id,
+    };
+  }
+}
+
+class VectorMetricParser extends ParsingInterface<VectorMetric> {
+  _parse(metric: VectorMetric): MetricModel {
+    return {
+      _string_attributes: metric.b,
+      data_type: metric.dt,
+      name: metric.n,
+      timestamp: metric.ts,
+      value: metric.v,
+    };
+  }
+}
+
+class VectorRrwebParser extends ParsingInterface<VectorLog> {
+  _parse(log: VectorLog): RrwebEventModel {
+    return {
+      ...mapObjectToKeyValuePairs(log.b),
+      _service: log.sv,
+      _source: log.r,
+      timestamp: log.ts,
+    };
+  }
+}
+
+// TODO: do this on the ingestor side ?
+export const extractApiKey = (log: VectorLog | VectorSpan | VectorMetric) => {
+  if (log.authorization?.includes('Bearer')) {
+    return log.authorization.split('Bearer ')[1];
+  }
+  return log.hdx_token;
+};
+
+export const vectorLogParser = new VectorLogParser();
+export const vectorMetricParser = new VectorMetricParser();
+export const vectorRrwebParser = new VectorRrwebParser();

packages/api/src/utils/logger.ts

@@ -0,0 +1,70 @@
+import _ from 'lodash';
+import expressWinston from 'express-winston';
+import winston, { addColors } from 'winston';
+import { getWinsonTransport } from '@hyperdx/node-opentelemetry';
+
+import {
+  APP_TYPE,
+  HYPERDX_API_KEY,
+  HYPERDX_INGESTOR_ENDPOINT,
+  IS_DEV,
+  IS_PROD,
+} from '../config';
+
+import type { IUser } from '../models/user';
+
+// LOCAL DEV ONLY
+addColors({
+  error: 'bold red',
+  warn: 'bold yellow',
+  info: 'white',
+  http: 'gray',
+  verbose: 'bold magenta',
+  debug: 'green',
+  silly: 'cyan',
+});
+
+const MAX_LEVEL = IS_PROD ? 'debug' : 'debug';
+const DEFAULT_FORMAT = IS_DEV
+  ? winston.format.combine(
+      winston.format.errors({ stack: true }),
+      winston.format.colorize({ all: true }),
+      winston.format.timestamp({ format: 'MM/DD/YY HH:MM:SS' }),
+      winston.format.printf(
+        info => `[${info.level}] ${info.timestamp} ${info.message}`,
+      ),
+    )
+  : winston.format.combine(
+      winston.format.errors({ stack: true }),
+      winston.format.json(),
+    );
+
+const hyperdxTransport = HYPERDX_API_KEY
+  ? getWinsonTransport(MAX_LEVEL, {
+      bufferSize: APP_TYPE === 'scheduled-task' ? 1 : 100,
+      ...(HYPERDX_INGESTOR_ENDPOINT && { baseUrl: HYPERDX_INGESTOR_ENDPOINT }),
+    })
+  : null;
+
+export const expressLogger = expressWinston.logger({
+  format: DEFAULT_FORMAT,
+  msg: IS_PROD
+    ? undefined
+    : 'HTTP {{res.statusCode}} {{req.method}} {{req.url}} {{res.responseTime}}ms',
+  transports: [
+    new winston.transports.Console(),
+    ...(hyperdxTransport ? [hyperdxTransport] : []),
+  ],
+  meta: IS_PROD,
+});
+
+const logger = winston.createLogger({
+  level: MAX_LEVEL,
+  format: DEFAULT_FORMAT,
+  transports: [
+    new winston.transports.Console(),
+    ...(hyperdxTransport ? [hyperdxTransport] : []),
+  ],
+});
+
+export default logger;

packages/api/src/utils/miner.ts

@@ -0,0 +1,31 @@
+import axios from 'axios';
+import ms from 'ms';
+
+import * as config from '../config';
+import logger from './logger';
+
+const MAX_LOG_LINES = 1e4;
+
+export const getLogsPatterns = async (
+  teamId: string,
+  lines: string[][],
+): Promise<{
+  patterns: Record<string, string>;
+  result: Record<string, string>;
+}> => {
+  if (lines.length > MAX_LOG_LINES) {
+    logger.error(`Too many log lines requested: ${lines.length}`);
+  }
+
+  return axios({
+    method: 'POST',
+    url: `${config.MINER_API_URL}/logs`,
+    data: {
+      team_id: teamId,
+      lines: lines.slice(0, MAX_LOG_LINES),
+    },
+    maxContentLength: Infinity,
+    maxBodyLength: Infinity,
+    timeout: ms('2 minute'),
+  }).then(response => response.data);
+};

packages/api/src/utils/passport.ts

@@ -0,0 +1,55 @@
+import passport from 'passport';
+import { Strategy as LocalStrategy } from 'passport-local';
+
+import logger from './logger';
+import User from '../models/user';
+import { findUserById } from '../controllers/user';
+
+import type { UserDocument } from '../models/user';
+
+passport.serializeUser(function (user, done) {
+  done(null, (user as any)._id);
+});
+
+passport.deserializeUser(function (id: string, done) {
+  findUserById(id)
+    .then(user => {
+      if (user == null) {
+        return done(new Error('User not found'));
+      }
+      done(null, user as UserDocument);
+    })
+    .catch(done);
+});
+
+// Use local passport strategy via passport-local-mongoose plugin
+const passportLocalMongooseAuthenticate = (User as any).authenticate();
+
+passport.use(
+  new LocalStrategy(
+    {
+      usernameField: 'email',
+    },
+    async function (username, password, done) {
+      try {
+        const { user, error } = await passportLocalMongooseAuthenticate(
+          username,
+          password,
+        );
+        if (error) {
+          logger.info({
+            message: `Login for "${username}" failed, ${error}"`,
+            type: 'user_login',
+            authType: 'password',
+          });
+        }
+        return done(null, user, error);
+      } catch (err) {
+        logger.error(`Login for "${username}" failed, error: ${err}"`);
+        return done(err);
+      }
+    },
+  ),
+);
+
+export default passport;

packages/api/src/utils/queue.ts

@@ -0,0 +1,22 @@
+export class LimitedSizeQueue<T = any> {
+  private readonly _limit: number;
+
+  private readonly _queue: T[];
+
+  constructor(limit: number) {
+    this._limit = limit;
+    this._queue = [];
+  }
+
+  enqueue(item: T) {
+    this._queue.push(item);
+    if (this._queue.length === this._limit + 1) {
+      // randomly remove an item
+      this._queue.splice(Math.floor(Math.random() * this._limit), 1);
+    }
+  }
+
+  toArray() {
+    return this._queue;
+  }
+}

packages/api/src/utils/rateLimiter.ts

@@ -0,0 +1,14 @@
+import rateLimit, { Options } from 'express-rate-limit';
+import RedisStore from 'rate-limit-redis';
+
+import redisClient from './redis';
+
+export default (config?: Partial<Options>) => async (req, rs, next) => {
+  return rateLimit({
+    ...config,
+    // Redis store configuration
+    store: new RedisStore({
+      sendCommand: (...args: string[]) => redisClient.sendCommand(args),
+    }),
+  })(req, rs, next);
+};

packages/api/src/utils/redis.ts

@@ -0,0 +1,15 @@
+import { createClient } from 'redis';
+import { serializeError } from 'serialize-error';
+
+import * as config from '../config';
+import logger from '../utils/logger';
+
+const client = createClient({
+  url: config.REDIS_URL,
+});
+
+client.on('error', (err: any) => {
+  logger.error('Redis error: ', serializeError(err));
+});
+
+export default client;

packages/api/src/utils/slack.ts

@@ -0,0 +1,10 @@
+import _ from 'lodash';
+import { IncomingWebhook } from '@slack/webhook';
+
+export function postMessageToWebhook(webhookUrl: string, message: any) {
+  const webhook = new IncomingWebhook(webhookUrl);
+  return webhook.send({
+    text: message.text,
+    blocks: message.blocks,
+  });
+}

packages/api/src/utils/validators.ts

@@ -0,0 +1,6 @@
+export const validatePassword = (password: string) => {
+  if (!password || password.length < 8 || password.length > 64) {
+    return false;
+  }
+  return true;
+};

packages/api/tsconfig.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "allowSyntheticDefaultImports": true,
+    "downlevelIteration": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "importHelpers": true,
+    "lib": ["ES2022", "dom"],
+    "module": "Node16",
+    "moduleResolution": "node",
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitAny": false,
+    "noImplicitReturns": false,
+    "noImplicitThis": false,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "outDir": "build",
+    "resolveJsonModule": true,
+    "skipLibCheck": false,
+    "sourceMap": true,
+    "strict": true,
+    "strictNullChecks": false,
+    "target": "ES2022"
+  },
+  "include": ["src"]
+}