diff --git a/.gitignore b/.gitignore index 35880ebf76..de01834eab 100644 --- a/.gitignore +++ b/.gitignore @@ -95,5 +95,8 @@ orbit/cmd/desktop/resource.syso orbit/cmd/orbit/manifest.xml orbit/cmd/orbit/resource.syso +# Residual files from osqueryd loadtests. +osquery_worker_*.jpg + # Residual files when building fleetd_tables extension. fleetd_tables_* diff --git a/tools/loadtest/osquery/macos/README.md b/tools/loadtest/osquery/macos/README.md new file mode 100644 index 0000000000..ba8bdbf24e --- /dev/null +++ b/tools/loadtest/osquery/macos/README.md @@ -0,0 +1,72 @@ +# Load test of osquery queries in macOS + +Following are the steps to load test osquery on macOS. +The purpose is to know the impact of Fleet provided queries on real devices. + +> At the time of writing the changes to add watchog logging needed for this script +> are under review: https://github.com/osquery/osquery/pull/8070. +> You will have to build osqueryd from source code. + +## Requirements + +- Install gnuplot and ripgrep: +```sh +brew install gnuplot ripgrep +``` +- Tooling to build osqueryd from source (at the time of writing this is needed), see https://osquery.readthedocs.io/en/stable/development/building/. + +## Build fleetd_tables + +We are going to use the fleetd tables as an extension so that it is also monitored by the watchdog. + +```sh +make fleetd-tables-darwin-universal +sudo cp fleetd_tables_darwin_universal.ext /usr/local/osquery_extensions/fleetd_tables.ext +echo "/usr/local/osquery_extensions/fleetd_tables.ext" > /tmp/extensions.load +``` + +## Run osquery + +> The following assumes a Fleet server instance running and listening at `localhost:8080`. + +```sh +sudo ENROLL_SECRET=<...> ./osquery/osqueryd \ + --verbose=true \ + --tls_dump=true \ + --pidfile=/Users/luk/osqueryd/osquery.pid \ + --database_path=/Users/luk/osqueryd/osquery.db \ + --logger_path=/Users/luk/osqueryd/osquery_log \ + --host_identifier=instance \ + --tls_server_certs=/Users/luk/fleetdm/git/fleet/tools/osquery/fleet.crt \ + --enroll_secret_env=ENROLL_SECRET \ + --tls_hostname=localhost:8080 \ + --enroll_tls_endpoint=/api/v1/osquery/enroll \ + --config_plugin=tls \ + --config_tls_endpoint=/api/v1/osquery/config \ + --config_refresh=60 \ + --disable_distributed=false \ + --distributed_plugin=tls \ + --distributed_tls_max_attempts=10 \ + --distributed_tls_read_endpoint=/api/v1/osquery/distributed/read \ + --distributed_tls_write_endpoint=/api/v1/osquery/distributed/write \ + --logger_plugin=tls,filesystem \ + --logger_tls_endpoint=/api/v1/osquery/log \ + --disable_carver=false \ + --carver_disable_function=false \ + --carver_start_endpoint=/api/v1/osquery/carve/begin \ + --carver_continue_endpoint=/api/v1/osquery/carve/block \ + --carver_block_size=2000000 \ + --extensions_autoload=/tmp/extensions.load + --allow_unsafe \ + --enable_watchdog_debug \ + --distributed_denylist_duration 0 \ + --enable_extensions_watchdog 2>&1 | tee /tmp/osqueryd.log +``` + +## Render CPU and memory usage + +```sh +./tools/loadtest/osquery/macos/gnuplot_osqueryd_cpu_memory.sh +``` + +> The horizontal red line is the configured CPU usage limit (hardcoded to `1200ms` in the `gnuplot_osqueryd_cpu_memory.sh`) diff --git a/tools/loadtest/osquery/macos/gnuplot_osqueryd_cpu_memory.sh b/tools/loadtest/osquery/macos/gnuplot_osqueryd_cpu_memory.sh new file mode 100755 index 0000000000..e69322c064 --- /dev/null +++ b/tools/loadtest/osquery/macos/gnuplot_osqueryd_cpu_memory.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +# Requirements: +# - ripgrep +# - gnuplot + +# Get PID of the osquery worker process. +osquery_pid=$(ps aux | grep -E "osqueryd\s*$" | awk {'print $2'}) + +# Extract CPU and memory data points from logs. +rg " (\d\d:\d\d:\d\d).* pid: $osquery_pid, cpu: (\d+)ms/\d+ms, memory: ([\d.]+)" -or '$1 $2 $3' /tmp/osqueryd.log > /tmp/osqueryd.dat + +# Generate gnuplot commands and render CPU and memory data points. +cat < gnuplot_commands.txt +set xdata time +set timefmt "%H:%M:%S" +set format x "%H:%M" +set key off +set xtics rotate by -45 +set terminal jpeg + +set title 'Memory (MB)' +set output 'osquery_worker_memory.jpg' +plot '/tmp/osqueryd.dat' using 1:3 with linespoints linetype 7 linewidth 2 title 'Memory (MB)' + +set title 'CPU' +set output 'osquery_worker_cpu.jpg' +set yrange [0:24000] +# +# The calculation used by osquery for CPU limit is: +# check_interval * number_of_physical_cores * (percent_cpu_limit / 100) +# where default values are: check_interval=3000ms, percent_cpu_limit=10%. +# On my Macbook with 4 physical core this gives 1200ms. +# +plot '/tmp/osqueryd.dat' using 1:2 with linespoints linetype 6 linewidth 2 title 'CPU', 1200 linecolor 1 +EOF + +gnuplot < gnuplot_commands.txt +rm gnuplot_commands.txt + +open osquery_worker_cpu.jpg osquery_worker_memory.jpg \ No newline at end of file