mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 21:37:41 +00:00
ab50b12a6b
plus the fix to reduce bloat in opamp agent logs
Users should be able to mount the custom otel collector config file and add/overrider receivers, processors and exporters
For example:
```
receivers:
hostmetrics:
collection_interval: 5s
scrapers:
cpu:
load:
memory:
disk:
filesystem:
network:
# override the default processors
processors:
batch:
send_batch_size: 10000
timeout: 10s
memory_limiter:
limit_mib: 2000
service:
pipelines:
metrics/hostmetrics:
receivers: [hostmetrics]
# attach existing processors
processors: [memory_limiter, batch]
# attach existing exporters
exporters: [clickhouse]
```
This will add a new `hostmetrics` receiver + `metrics/hostmetrics` pipeline and update existing `batch` + `memory_limiter` processors
WARNING: This feature is still in beta, and future updates may change how it works, potentially affecting compatibility
Ref: HDX-1865
77 lines
3.5 KiB
YAML
77 lines
3.5 KiB
YAML
# Receivers are now configured dynamically in opampController.ts
|
|
processors:
|
|
transform:
|
|
log_statements:
|
|
- context: log
|
|
error_mode: ignore
|
|
statements:
|
|
# JSON parsing: Extends log attributes with the fields from structured log body content, either as an OTEL map or
|
|
# as a string containing JSON content.
|
|
- set(log.cache, ExtractPatterns(log.body, "(?P<0>(\\{.*\\}))")) where IsString(log.body)
|
|
- merge_maps(log.attributes, ParseJSON(log.cache["0"]), "upsert") where IsMap(log.cache)
|
|
- flatten(log.attributes) where IsMap(log.cache)
|
|
- merge_maps(log.attributes, log.body, "upsert") where IsMap(log.body)
|
|
- context: log
|
|
error_mode: ignore
|
|
conditions:
|
|
- severity_number == 0 and severity_text == ""
|
|
statements:
|
|
# Infer: extract the first log level keyword from the first 256 characters of the body
|
|
- set(log.cache["substr"], log.body.string) where Len(log.body.string) < 256
|
|
- set(log.cache["substr"], Substring(log.body.string, 0, 256)) where Len(log.body.string) >= 256
|
|
- set(log.cache, ExtractPatterns(log.cache["substr"], "(?i)(?P<0>(alert|crit|emerg|fatal|error|err|warn|notice|debug|dbug|trace))"))
|
|
# Infer: detect FATAL
|
|
- set(log.severity_number, SEVERITY_NUMBER_FATAL) where IsMatch(log.cache["0"], "(?i)(alert|crit|emerg|fatal)")
|
|
- set(log.severity_text, "fatal") where log.severity_number == SEVERITY_NUMBER_FATAL
|
|
# Infer: detect ERROR
|
|
- set(log.severity_number, SEVERITY_NUMBER_ERROR) where IsMatch(log.cache["0"], "(?i)(error|err)")
|
|
- set(log.severity_text, "error") where log.severity_number == SEVERITY_NUMBER_ERROR
|
|
# Infer: detect WARN
|
|
- set(log.severity_number, SEVERITY_NUMBER_WARN) where IsMatch(log.cache["0"], "(?i)(warn|notice)")
|
|
- set(log.severity_text, "warn") where log.severity_number == SEVERITY_NUMBER_WARN
|
|
# Infer: detect DEBUG
|
|
- set(log.severity_number, SEVERITY_NUMBER_DEBUG) where IsMatch(log.cache["0"], "(?i)(debug|dbug)")
|
|
- set(log.severity_text, "debug") where log.severity_number == SEVERITY_NUMBER_DEBUG
|
|
# Infer: detect TRACE
|
|
- set(log.severity_number, SEVERITY_NUMBER_TRACE) where IsMatch(log.cache["0"], "(?i)(trace)")
|
|
- set(log.severity_text, "trace") where log.severity_number == SEVERITY_NUMBER_TRACE
|
|
# Infer: else
|
|
- set(log.severity_text, "info") where log.severity_number == 0
|
|
- set(log.severity_number, SEVERITY_NUMBER_INFO) where
|
|
log.severity_number == 0
|
|
- context: log
|
|
error_mode: ignore
|
|
statements:
|
|
# Normalize the severity_text case
|
|
- set(log.severity_text, ConvertCase(log.severity_text, "lower"))
|
|
resourcedetection:
|
|
detectors:
|
|
- env
|
|
- system
|
|
- docker
|
|
timeout: 5s
|
|
override: false
|
|
batch:
|
|
memory_limiter:
|
|
# 80% of maximum memory up to 2G
|
|
limit_mib: 1500
|
|
# 25% of limit up to 2G
|
|
spike_limit_mib: 512
|
|
check_interval: 5s
|
|
# Connectors and exporters are now configured dynamically in opampController.ts
|
|
extensions:
|
|
health_check:
|
|
endpoint: :13133
|
|
service:
|
|
telemetry:
|
|
metrics:
|
|
readers:
|
|
- pull:
|
|
exporter:
|
|
prometheus:
|
|
host: '0.0.0.0'
|
|
port: 8888
|
|
logs:
|
|
level: ${HYPERDX_LOG_LEVEL}
|
|
extensions: [health_check]
|
|
# Pipelines are now configured dynamically in opampController.ts
|