From 389bb3a027f23f0faa63de0c37127261c224845e Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:01:44 -0700 Subject: [PATCH] feat: support `HYPERDX_LOG_LEVEL` env var (#66) --- .changeset/calm-plums-hug.md | 6 ++ .env | 1 + .kodiak.toml | 98 ++++++++++++++++++++++++++++++ README.md | 8 +++ docker-compose.dev.yml | 11 +++- docker-compose.yml | 11 +++- docker/hostmetrics/config.dev.yaml | 2 +- docker/otel-collector/config.yaml | 2 +- packages/api/src/config.ts | 1 + packages/api/src/utils/logger.ts | 4 +- packages/miner/src/main.py | 11 +++- 11 files changed, 147 insertions(+), 8 deletions(-) create mode 100644 .changeset/calm-plums-hug.md diff --git a/.changeset/calm-plums-hug.md b/.changeset/calm-plums-hug.md new file mode 100644 index 00000000..bc1acff8 --- /dev/null +++ b/.changeset/calm-plums-hug.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +feat: support HYPERDX_LOG_LEVEL env var diff --git a/.env b/.env index 443ea8d9..6ba63314 100644 --- a/.env +++ b/.env @@ -7,5 +7,6 @@ HYPERDX_API_PORT=8000 HYPERDX_API_URL=http://localhost HYPERDX_APP_PORT=8080 HYPERDX_APP_URL=http://localhost +HYPERDX_LOG_LEVEL=debug OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 # port is fixed diff --git a/.kodiak.toml b/.kodiak.toml index d9914dfa..11ae67a6 100644 --- a/.kodiak.toml +++ b/.kodiak.toml @@ -1 +1,99 @@ version = 1 + +[merge] +# Label to enable Kodiak to merge a PR. +automerge_label = "automerge" + +# When disabled, Kodiak will immediately attempt to merge any PR that passes all +# GitHub branch protection requirements. +require_automerge_label = true + +# If you're using the "Require signed commits" GitHub Branch Protection setting +# to require commit signatures, "merge" or "squash" are the only compatible options. "rebase" will cause Kodiak to raise a configuration error. +method = "squash" # default: "merge", options: "merge", "squash", "rebase" + +# Once a PR is merged, delete the branch. This option behaves like the GitHub +# repository setting "Automatically delete head branches", which automatically +# deletes head branches after pull requests are merged. +delete_branch_on_merge = true # default: false + +# If there is a merge conflict, make a comment on the PR and remove the +# automerge label. This option only applies when `merge.require_automerge_label` +# is enabled. +notify_on_conflict = true # default: true + +# Don't wait for in-progress status checks on a PR to finish before updating the +# branch. +optimistic_updates = true # default: true + +# If a PR is passing all checks and is able to be merged, merge it without +# placing it in the merge queue. This option adds some unfairness where PRs +# waiting in the queue the longest are not served first. +prioritize_ready_to_merge = true # default: false + +# Never merge a PR. This option can be used with `update.always` to +# automatically update a PR without merging. +do_not_merge = false # default: false + +[merge.message] +# By default (`"github_default"`), GitHub uses the title of a PR's first commit +# for the merge commit title. `"pull_request_title"` uses the PR title for the +# merge commit. +title = "pull_request_title" # default: "github_default", options: "github_default", "pull_request_title" + +# By default (`"github_default"`), GitHub combines the titles of a PR's commits +# to create the body text of a merge commit. `"pull_request_body"` uses the +# content of the PR to generate the body content while `"empty"` sets an empty +# body. +body = "pull_request_body" # default: "github_default", options: "github_default", "pull_request_body", "empty" + +# Append the Pull Request URL to the merge message. Makes navigating to the PR +# from the commit easier. +#### NOTE: 'true' required for benchmarks in CI: +include_pull_request_url = false # default: false + +# Add the PR number to the merge commit title. This setting replicates GitHub's +# behavior of automatically adding the PR number to the title of merges created +# through the UI. This option only applies when `merge.message.title` does not +# equal `"github_default"`. +### NOTE: if this is set to true github links to unrelated OSS issues, which is confusing +include_pr_number = true # default: true + +# Control the text used in the merge commit. The GitHub default is markdown, but +# `"plain_text"` or `"html"` can be used to render the pull request body as text +# or HTML. This option only applies when `merge.message.body = "pull_request_body"`. +body_type = "markdown" # default: "markdown", options: "plain_text", "markdown", "html" + +# Strip HTML comments (``) from merge commit body. +# This setting is useful for stripping HTML comments created by PR templates. +# This option only applies when `merge.message.body_type = "markdown"`. +strip_html_comments = true # default: false + +# Remove all content before the configured string in the pull request body. +# This setting is useful when we want to include only a part of the pull request +# description as the commit message. +# This option only applies when `merge.message.body_type = "markdown"`. +cut_body_before = "" + +include_coauthors = true + +[update] + +# Update a PR whenever out of date with the base branch. The PR will be updated +# regardless of merge requirements (e.g. failing status checks, missing reviews, +# blacklist labels). +# +# Kodiak will only update PRs with the `merge.automerge_label` label or if +# `update.require_automerge_label = false`. +# +# When enabled, _Kodiak will not be able to efficiently update PRs._ If you have +# multiple PRs against a target like `master`, any time a commit is added to +# `master` _all_ of those PRs against `master` will update. For `N` PRs against +# a target you will see at least `N(N-1)/2` updates. If this configuration +# option was disabled you would only see at least `N-1` updates. +always = false # default: false + +# When enabled, Kodiak will only update PRs that have an automerge label +# (configured via `merge.automerge_label`). When disable, Kodiak will update any +# PR. This option only applies when `update.always = true`. +require_automerge_label = true # default: true diff --git a/README.md b/README.md index e0b4e710..153f25aa 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,14 @@ comprehensive documentation on how we balance between cloud-only and open source features in the future. In the meantime, we're highly aligned with Gitlab's [stewardship model](https://handbook.gitlab.com/handbook/company/stewardship/). +## Frequently Asked Questions + +#### How to suppress all logs from HyperDX itself ? + +To suppress logs of a service, you can comment out the `HYPERDX_API_KEY` +environment variable in the docker-compose.yml file. The alternative is to set +the `HYPERDX_LOG_LEVEL` environment variable to 'error' to only log errors. + ## Contact - [Open an Issue](https://github.com/hyperdxio/hyperdx/issues/new) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index daa7c429..ab2893ad 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -9,8 +9,9 @@ services: environment: HYPERDX_API_KEY: ${HYPERDX_API_KEY} HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1 + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 - OTEL_LOG_LEVEL: ERROR + OTEL_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} OTEL_SERVICE_NAME: hdx-oss-dev-miner volumes: - ./packages/miner/src:/app/src @@ -27,6 +28,7 @@ services: - ./docker/hostmetrics/config.dev.yaml:/etc/otelcol-contrib/config.yaml environment: HYPERDX_API_KEY: ${HYPERDX_API_KEY} + OTEL_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} OTEL_SERVICE_NAME: hostmetrics restart: always networks: @@ -44,7 +46,7 @@ services: - 8686:8686 # healthcheck environment: RUST_BACKTRACE: full - VECTOR_LOG: debug + VECTOR_LOG: ${HYPERDX_LOG_LEVEL} VECTOR_OPENSSL_LEGACY_PROVIDER: false restart: always networks: @@ -72,6 +74,8 @@ services: build: context: ./docker/otel-collector target: dev + environment: + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml ports: @@ -100,6 +104,7 @@ services: CLICKHOUSE_PASSWORD: aggregator CLICKHOUSE_USER: aggregator FRONTEND_URL: 'http://localhost:8080' # need to be localhost (CORS) + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} MONGO_URI: 'mongodb://db:27017/hyperdx' NODE_ENV: development PORT: 8001 @@ -132,6 +137,7 @@ services: HDX_NODE_BETA_MODE: 0 HDX_NODE_CONSOLE_CAPTURE: 1 HYPERDX_API_KEY: ${HYPERDX_API_KEY} + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} INGESTOR_API_URL: 'http://ingestor:8002' MINER_API_URL: 'http://miner:5123' MONGO_URI: 'mongodb://db:27017/hyperdx' @@ -169,6 +175,7 @@ services: HDX_NODE_BETA_MODE: 1 HDX_NODE_CONSOLE_CAPTURE: 1 HYPERDX_API_KEY: ${HYPERDX_API_KEY} + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} INGESTOR_API_URL: 'http://ingestor:8002' MINER_API_URL: 'http://miner:5123' MONGO_URI: 'mongodb://db:27017/hyperdx' diff --git a/docker-compose.yml b/docker-compose.yml index de4b93d0..df92b18f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,9 @@ services: environment: HYPERDX_API_KEY: ${HYPERDX_API_KEY} HYPERDX_ENABLE_ADVANCED_NETWORK_CAPTURE: 1 + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 - OTEL_LOG_LEVEL: ERROR + OTEL_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} OTEL_SERVICE_NAME: hdx-oss-miner ports: - 5123:5123 @@ -18,6 +19,7 @@ services: container_name: hdx-oss-hostmetrics environment: HYPERDX_API_KEY: ${HYPERDX_API_KEY} + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} OTEL_SERVICE_NAME: hostmetrics restart: always networks: @@ -32,7 +34,7 @@ services: - 8686:8686 # healthcheck environment: RUST_BACKTRACE: full - VECTOR_LOG: debug + VECTOR_LOG: ${HYPERDX_LOG_LEVEL} VECTOR_OPENSSL_LEGACY_PROVIDER: false restart: always networks: @@ -58,6 +60,8 @@ services: otel-collector: image: ${IMAGE_NAME}:${IMAGE_VERSION}-otel-collector container_name: hdx-oss-otel-collector + environment: + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} ports: - '13133:13133' # health_check extension - '1888:1888' # pprof extension @@ -81,6 +85,7 @@ services: CLICKHOUSE_PASSWORD: aggregator CLICKHOUSE_USER: aggregator FRONTEND_URL: ${HYPERDX_APP_URL}:${HYPERDX_APP_PORT} # need to be localhost (CORS) + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} MONGO_URI: 'mongodb://db:27017/hyperdx' NODE_ENV: development PORT: 8001 @@ -108,6 +113,7 @@ services: HDX_NODE_BETA_MODE: 0 HDX_NODE_CONSOLE_CAPTURE: 1 HYPERDX_API_KEY: ${HYPERDX_API_KEY} + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} INGESTOR_API_URL: 'http://ingestor:8002' MINER_API_URL: 'http://miner:5123' MONGO_URI: 'mongodb://db:27017/hyperdx' @@ -140,6 +146,7 @@ services: HDX_NODE_BETA_MODE: 1 HDX_NODE_CONSOLE_CAPTURE: 1 HYPERDX_API_KEY: ${HYPERDX_API_KEY} + HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL} INGESTOR_API_URL: 'http://ingestor:8002' MINER_API_URL: 'http://miner:5123' MONGO_URI: 'mongodb://db:27017/hyperdx' diff --git a/docker/hostmetrics/config.dev.yaml b/docker/hostmetrics/config.dev.yaml index 4528f76e..8af3c51f 100644 --- a/docker/hostmetrics/config.dev.yaml +++ b/docker/hostmetrics/config.dev.yaml @@ -21,7 +21,7 @@ receivers: network: exporters: logging: - loglevel: debug + loglevel: ${env:HYPERDX_LOG_LEVEL} otlphttp: endpoint: 'http://otel-collector:4318' headers: diff --git a/docker/otel-collector/config.yaml b/docker/otel-collector/config.yaml index dbadc635..ad1a6626 100644 --- a/docker/otel-collector/config.yaml +++ b/docker/otel-collector/config.yaml @@ -39,7 +39,7 @@ processors: check_interval: 5s exporters: logging: - loglevel: debug + loglevel: ${env:HYPERDX_LOG_LEVEL} logzio/traces: account_token: 'X' # required but we don't use it endpoint: 'http://ingestor:8002?hdx_platform=otel-traces' diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 019dee85..771d8abe 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -11,6 +11,7 @@ export const COOKIE_DOMAIN = env.COOKIE_DOMAIN as string; // prod ONLY export const EXPRESS_SESSION_SECRET = env.EXPRESS_SESSION_SECRET as string; export const FRONTEND_URL = env.FRONTEND_URL as string; export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string; +export const HYPERDX_LOG_LEVEL = env.HYPERDX_LOG_LEVEL as string; export const INGESTOR_API_URL = env.INGESTOR_API_URL as string; export const IS_CI = NODE_ENV === 'ci'; export const IS_DEV = NODE_ENV === 'development'; diff --git a/packages/api/src/utils/logger.ts b/packages/api/src/utils/logger.ts index 87da16ac..48c16504 100644 --- a/packages/api/src/utils/logger.ts +++ b/packages/api/src/utils/logger.ts @@ -5,6 +5,7 @@ import winston, { addColors } from 'winston'; import { APP_TYPE, HYPERDX_API_KEY, + HYPERDX_LOG_LEVEL, INGESTOR_API_URL, IS_PROD, } from '../config'; @@ -20,7 +21,7 @@ addColors({ silly: 'cyan', }); -const MAX_LEVEL = IS_PROD ? 'debug' : 'debug'; +const MAX_LEVEL = HYPERDX_LOG_LEVEL ?? 'debug'; const DEFAULT_FORMAT = winston.format.combine( winston.format.errors({ stack: true }), winston.format.json(), @@ -34,6 +35,7 @@ const hyperdxTransport = HYPERDX_API_KEY : null; export const expressLogger = expressWinston.logger({ + level: MAX_LEVEL, format: DEFAULT_FORMAT, msg: IS_PROD ? undefined diff --git a/packages/miner/src/main.py b/packages/miner/src/main.py index 04933e40..6d9a4e1a 100644 --- a/packages/miner/src/main.py +++ b/packages/miner/src/main.py @@ -2,6 +2,7 @@ from typing import List import hashlib import json import logging +import os import time from drain3 import TemplateMiner @@ -16,8 +17,16 @@ API_VERSION = "0.0.1" app = FastAPI() + +def get_logging_level(): + try: + return getattr(logging, os.environ.get("HYPERDX_LOG_LEVEL", "DEBUG").upper()) + except Exception: + return logging.DEBUG + + logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(get_logging_level()) def get_template_miner():