mirror of
https://github.com/hyperdxio/hyperdx
synced 2026-04-21 13:37:15 +00:00
feat: normalize the text case of severity text (#834)
This commit is contained in:
Dan Hable
GitHub
parent
115540734c
commit
d9ab75710e
38 changed files with 329 additions and 85 deletions
|
|
@ -63,6 +63,11 @@ processors:
|
|||
# Infer: else
|
||||
- set(log.severity_text, "info") where log.severity_number == 0
|
||||
- set(log.severity_number, SEVERITY_NUMBER_INFO) where log.severity_number == 0
|
||||
- context: log
|
||||
error_mode: ignore
|
||||
statements:
|
||||
# Normalize the severity_text case
|
||||
- set(log.severity_text, ConvertCase(log.severity_text, "lower"))
|
||||
resourcedetection:
|
||||
detectors:
|
||||
- env
|
||||
|
|
|
|||
|
|
@ -29,22 +29,29 @@ bats hdx-1453-auto-parse-json.bats
|
|||
## Test Structure
|
||||
|
||||
- `*.bats` - Test files written in Bats
|
||||
- `setup_suite.bash` - Contains global setup_suite and teardown_suite functions that run once for the entire test suite
|
||||
- `data/` - Test data used by the tests
|
||||
- `test_helpers/` - Utility functions for the tests
|
||||
- `docker-compose.yaml` - Docker Compose configuration for the test environment
|
||||
|
||||
The test suite uses Bats' `setup_suite` and `teardown_suite` hooks to initialize the environment only once, regardless of how many test files are run. This optimizes test execution by:
|
||||
|
||||
1. Validating the environment once
|
||||
2. Starting Docker containers once at the beginning of the test suite
|
||||
3. Cleaning up containers once at the end of the test suite
|
||||
|
||||
## Debugging
|
||||
|
||||
If you need to debug the tests, you can set the `SKIP_CLEANUP` environment variable to prevent the Docker containers from being torn down after the tests complete:
|
||||
|
||||
```bash
|
||||
SKIP_CLEANUP=1 bats hdx-1453-auto-parse-json.bats
|
||||
SKIP_CLEANUP=1 bats *.bats
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```bash
|
||||
SKIP_CLEANUP=true bats hdx-1453-auto-parse-json.bats
|
||||
SKIP_CLEANUP=true bats *.bats
|
||||
```
|
||||
|
||||
With `SKIP_CLEANUP` enabled, the test containers will remain running after the tests complete, allowing you to inspect logs, connect to the containers, and debug issues.
|
||||
|
|
|
|||
|
|
@ -3,29 +3,19 @@
|
|||
load 'test_helpers/utilities.bash'
|
||||
load 'test_helpers/assertions.bash'
|
||||
|
||||
setup_file() {
|
||||
validate_env
|
||||
docker compose up --build --detach
|
||||
wait_for_ready "otel-collector"
|
||||
}
|
||||
|
||||
teardown_file() {
|
||||
attempt_env_cleanup
|
||||
}
|
||||
|
||||
@test "JSON string body content should be parsed and stored as log attributes" {
|
||||
@test "HDX-1453: JSON string body content should be parsed and stored as log attributes" {
|
||||
emit_otel_data "http://localhost:4318" "data/auto-parse/json-string"
|
||||
sleep 1
|
||||
assert_test_data "data/auto-parse/json-string"
|
||||
}
|
||||
|
||||
@test "OTEL map content should be stored as log attributes" {
|
||||
@test "HDX-1453: OTEL map content should be stored as log attributes" {
|
||||
emit_otel_data "http://localhost:4318" "data/auto-parse/otel-map"
|
||||
sleep 1
|
||||
assert_test_data "data/auto-parse/otel-map"
|
||||
}
|
||||
|
||||
@test "all other content should skip storing values in log attributes" {
|
||||
@test "HDX-1453: all other content should skip storing values in log attributes" {
|
||||
emit_otel_data "http://localhost:4318" "data/auto-parse/default"
|
||||
sleep 1
|
||||
assert_test_data "data/auto-parse/default"
|
||||
|
|
@ -1 +0,0 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'hdx-1514' AND ResourceAttributes['test-id'] = 'infer-debug' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -1 +0,0 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'hdx-1514' AND ResourceAttributes['test-id'] = 'infer-error' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -1 +0,0 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'hdx-1514' AND ResourceAttributes['test-id'] = 'infer-fatal' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -1 +0,0 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'hdx-1514' AND ResourceAttributes['test-id'] = 'infer-info' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -1 +0,0 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'hdx-1514' AND ResourceAttributes['test-id'] = 'infer-trace' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -1 +0,0 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'hdx-1514' AND ResourceAttributes['test-id'] = 'infer-warn' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -1 +0,0 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'hdx-1514' AND ResourceAttributes['test-id'] = 'skip-infer' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -0,0 +1 @@
|
|||
SELECT SeverityText FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'normalize-severity' AND ResourceAttributes['test-id'] = 'text-case' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
"trace"
|
||||
"trace"
|
||||
"trace"
|
||||
"debug"
|
||||
"debug"
|
||||
"debug"
|
||||
"info"
|
||||
"info"
|
||||
"info"
|
||||
"warn"
|
||||
"warn"
|
||||
"warn"
|
||||
"warning"
|
||||
"warning"
|
||||
"error"
|
||||
"error"
|
||||
"error"
|
||||
"fatal"
|
||||
"fatal"
|
||||
"fatal"
|
||||
"critical"
|
||||
"critical"
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
{
|
||||
"resourceLogs": [
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "suite-id",
|
||||
"value": {
|
||||
"stringValue": "normalize-severity"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "test-id",
|
||||
"value": {
|
||||
"stringValue": "text-case"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeLogs": [
|
||||
{
|
||||
"scope": {},
|
||||
"logRecords": [
|
||||
{
|
||||
"timeUnixNano": "1901999580000000000",
|
||||
"severity_number": 1,
|
||||
"severity_text": "TrAcE",
|
||||
"body": {
|
||||
"stringValue": "trace log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000001",
|
||||
"severity_number": 1,
|
||||
"severity_text": "trace",
|
||||
"body": {
|
||||
"stringValue": "trace log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000002",
|
||||
"severity_number": 1,
|
||||
"severity_text": "TRACE",
|
||||
"body": {
|
||||
"stringValue": "trace log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000003",
|
||||
"severity_number": 5,
|
||||
"severity_text": "DeBuG",
|
||||
"body": {
|
||||
"stringValue": "debug log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000004",
|
||||
"severity_number": 5,
|
||||
"severity_text": "debug",
|
||||
"body": {
|
||||
"stringValue": "debug log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000005",
|
||||
"severity_number": 5,
|
||||
"severity_text": "DEBUG",
|
||||
"body": {
|
||||
"stringValue": "debug log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000006",
|
||||
"severity_number": 9,
|
||||
"severity_text": "InFo",
|
||||
"body": {
|
||||
"stringValue": "info log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000007",
|
||||
"severity_number": 9,
|
||||
"severity_text": "info",
|
||||
"body": {
|
||||
"stringValue": "info log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000008",
|
||||
"severity_number": 9,
|
||||
"severity_text": "INFO",
|
||||
"body": {
|
||||
"stringValue": "info log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000009",
|
||||
"severity_number": 13,
|
||||
"severity_text": "WaRn",
|
||||
"body": {
|
||||
"stringValue": "warn log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000010",
|
||||
"severity_number": 13,
|
||||
"severity_text": "warn",
|
||||
"body": {
|
||||
"stringValue": "warn log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000011",
|
||||
"severity_number": 13,
|
||||
"severity_text": "WARN",
|
||||
"body": {
|
||||
"stringValue": "warn log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000012",
|
||||
"severity_number": 13,
|
||||
"severity_text": "warning",
|
||||
"body": {
|
||||
"stringValue": "warn log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000013",
|
||||
"severity_number": 13,
|
||||
"severity_text": "WARNING",
|
||||
"body": {
|
||||
"stringValue": "warn log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000014",
|
||||
"severity_number": 17,
|
||||
"severity_text": "ErRoR",
|
||||
"body": {
|
||||
"stringValue": "error log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000015",
|
||||
"severity_number": 17,
|
||||
"severity_text": "error",
|
||||
"body": {
|
||||
"stringValue": "error log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000016",
|
||||
"severity_number": 17,
|
||||
"severity_text": "ERROR",
|
||||
"body": {
|
||||
"stringValue": "error log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000017",
|
||||
"severity_number": 21,
|
||||
"severity_text": "FaTaL",
|
||||
"body": {
|
||||
"stringValue": "fatal log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000018",
|
||||
"severity_number": 21,
|
||||
"severity_text": "fatal",
|
||||
"body": {
|
||||
"stringValue": "fatal log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000019",
|
||||
"severity_number": 21,
|
||||
"severity_text": "FATAL",
|
||||
"body": {
|
||||
"stringValue": "fatal log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000020",
|
||||
"severity_number": 21,
|
||||
"severity_text": "critical",
|
||||
"body": {
|
||||
"stringValue": "fatal log message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"timeUnixNano": "1901999580000000021",
|
||||
"severity_number": 21,
|
||||
"severity_text": "CRITICAL",
|
||||
"body": {
|
||||
"stringValue": "fatal log message"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-debug' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
{
|
||||
"key": "suite-id",
|
||||
"value": {
|
||||
"stringValue": "hdx-1514"
|
||||
"stringValue": "severity-inference"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -0,0 +1 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-error' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
{
|
||||
"key": "suite-id",
|
||||
"value": {
|
||||
"stringValue": "hdx-1514"
|
||||
"stringValue": "severity-inference"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -0,0 +1 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-fatal' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
{
|
||||
"key": "suite-id",
|
||||
"value": {
|
||||
"stringValue": "hdx-1514"
|
||||
"stringValue": "severity-inference"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -0,0 +1 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-info' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
{
|
||||
"key": "suite-id",
|
||||
"value": {
|
||||
"stringValue": "hdx-1514"
|
||||
"stringValue": "severity-inference"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -0,0 +1 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-trace' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
{
|
||||
"key": "suite-id",
|
||||
"value": {
|
||||
"stringValue": "hdx-1514"
|
||||
"stringValue": "severity-inference"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -0,0 +1 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'infer-warn' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
{
|
||||
"key": "suite-id",
|
||||
"value": {
|
||||
"stringValue": "hdx-1514"
|
||||
"stringValue": "severity-inference"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -0,0 +1 @@
|
|||
SELECT SeverityText, SeverityNumber, Body FROM otel_logs WHERE ResourceAttributes['suite-id'] = 'severity-inference' AND ResourceAttributes['test-id'] = 'skip-infer' ORDER BY TimestampTime FORMAT CSV
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
{
|
||||
"key": "suite-id",
|
||||
"value": {
|
||||
"stringValue": "hdx-1514"
|
||||
"stringValue": "severity-inference"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
#!/usr/bin/env bats
|
||||
|
||||
load 'test_helpers/utilities.bash'
|
||||
load 'test_helpers/assertions.bash'
|
||||
|
||||
setup_file() {
|
||||
validate_env
|
||||
docker compose up --build --detach
|
||||
wait_for_ready "otel-collector"
|
||||
}
|
||||
|
||||
teardown_file() {
|
||||
attempt_env_cleanup
|
||||
}
|
||||
|
||||
@test "should infer fatal log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/hdx-1514-inference/infer-fatal"
|
||||
sleep 1
|
||||
assert_test_data "data/hdx-1514-inference/infer-fatal"
|
||||
}
|
||||
|
||||
@test "should infer error log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/hdx-1514-inference/infer-error"
|
||||
sleep 1
|
||||
assert_test_data "data/hdx-1514-inference/infer-error"
|
||||
}
|
||||
|
||||
@test "should infer warn log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/hdx-1514-inference/infer-warn"
|
||||
sleep 1
|
||||
assert_test_data "data/hdx-1514-inference/infer-warn"
|
||||
}
|
||||
|
||||
@test "should infer debug log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/hdx-1514-inference/infer-debug"
|
||||
sleep 1
|
||||
assert_test_data "data/hdx-1514-inference/infer-debug"
|
||||
}
|
||||
|
||||
@test "should infer trace log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/hdx-1514-inference/infer-trace"
|
||||
sleep 1
|
||||
assert_test_data "data/hdx-1514-inference/infer-trace"
|
||||
}
|
||||
|
||||
@test "should infer info log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/hdx-1514-inference/infer-info"
|
||||
sleep 1
|
||||
assert_test_data "data/hdx-1514-inference/infer-info"
|
||||
}
|
||||
|
||||
@test "should skip inference if severity values are defined on the input" {
|
||||
emit_otel_data "http://localhost:4318" "data/hdx-1514-inference/skip-infer"
|
||||
sleep 1
|
||||
assert_test_data "data/hdx-1514-inference/skip-infer"
|
||||
}
|
||||
10
smoke-tests/otel-collector/normalize-severity.bats
Normal file
10
smoke-tests/otel-collector/normalize-severity.bats
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
#!/usr/bin/env bats
|
||||
|
||||
load 'test_helpers/utilities.bash'
|
||||
load 'test_helpers/assertions.bash'
|
||||
|
||||
@test "HDX-1422: normalize the text case for severity text" {
|
||||
emit_otel_data "http://localhost:4318" "data/normalize-severity/text-case"
|
||||
sleep 1
|
||||
assert_test_data "data/normalize-severity/text-case"
|
||||
}
|
||||
14
smoke-tests/otel-collector/setup_suite.bash
Normal file
14
smoke-tests/otel-collector/setup_suite.bash
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/env bats
|
||||
|
||||
load 'test_helpers/utilities.bash'
|
||||
load 'test_helpers/assertions.bash'
|
||||
|
||||
setup_suite() {
|
||||
validate_env
|
||||
docker compose up --build --detach
|
||||
wait_for_ready "otel-collector"
|
||||
}
|
||||
|
||||
teardown_suite() {
|
||||
attempt_env_cleanup
|
||||
}
|
||||
46
smoke-tests/otel-collector/severity-inference.bats
Normal file
46
smoke-tests/otel-collector/severity-inference.bats
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
#!/usr/bin/env bats
|
||||
|
||||
load 'test_helpers/utilities.bash'
|
||||
load 'test_helpers/assertions.bash'
|
||||
|
||||
@test "HDX-1514: should infer fatal log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/severity-inference/infer-fatal"
|
||||
sleep 1
|
||||
assert_test_data "data/severity-inference/infer-fatal"
|
||||
}
|
||||
|
||||
@test "HDX-1514: should infer error log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/severity-inference/infer-error"
|
||||
sleep 1
|
||||
assert_test_data "data/severity-inference/infer-error"
|
||||
}
|
||||
|
||||
@test "HDX-1514: should infer warn log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/severity-inference/infer-warn"
|
||||
sleep 1
|
||||
assert_test_data "data/severity-inference/infer-warn"
|
||||
}
|
||||
|
||||
@test "HDX-1514: should infer debug log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/severity-inference/infer-debug"
|
||||
sleep 1
|
||||
assert_test_data "data/severity-inference/infer-debug"
|
||||
}
|
||||
|
||||
@test "HDX-1514: should infer trace log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/severity-inference/infer-trace"
|
||||
sleep 1
|
||||
assert_test_data "data/severity-inference/infer-trace"
|
||||
}
|
||||
|
||||
@test "HDX-1514: should infer info log level" {
|
||||
emit_otel_data "http://localhost:4318" "data/severity-inference/infer-info"
|
||||
sleep 1
|
||||
assert_test_data "data/severity-inference/infer-info"
|
||||
}
|
||||
|
||||
@test "HDX-1514: should skip inference if severity values are defined on the input" {
|
||||
emit_otel_data "http://localhost:4318" "data/severity-inference/skip-infer"
|
||||
sleep 1
|
||||
assert_test_data "data/severity-inference/skip-infer"
|
||||
}
|
||||
Loading…
Reference in a new issue