Add tooling to loadtest osqueryd in macOS (#12518)

#10292

This is an internal tool (for macOS) to load test the impact of queries
in osquery.

Sample output of script:

![osquery_worker_memory](https://github.com/fleetdm/fleet/assets/2073526/60bd10c3-270c-49b4-89f4-7b280b48b679)

![osquery_worker_cpu](https://github.com/fleetdm/fleet/assets/2073526/bf918359-914e-44f5-960e-ad252f78b7e0)

.gitignore

@@ -95,5 +95,8 @@ orbit/cmd/desktop/resource.syso
 orbit/cmd/orbit/manifest.xml
 orbit/cmd/orbit/resource.syso
 
+# Residual files from osqueryd loadtests.
+osquery_worker_*.jpg
+
 # Residual files when building fleetd_tables extension.
 fleetd_tables_*

tools/loadtest/osquery/macos/README.md

@@ -0,0 +1,72 @@
+# Load test of osquery queries in macOS
+
+Following are the steps to load test osquery on macOS.
+The purpose is to know the impact of Fleet provided queries on real devices.
+
+> At the time of writing the changes to add watchog logging needed for this script
+> are under review: https://github.com/osquery/osquery/pull/8070.
+> You will have to build osqueryd from source code.
+
+## Requirements
+
+- Install gnuplot and ripgrep:
+```sh
+brew install gnuplot ripgrep
+```
+- Tooling to build osqueryd from source (at the time of writing this is needed), see https://osquery.readthedocs.io/en/stable/development/building/.
+
+## Build fleetd_tables
+
+We are going to use the fleetd tables as an extension so that it is also monitored by the watchdog.
+
+```sh
+make fleetd-tables-darwin-universal
+sudo cp fleetd_tables_darwin_universal.ext /usr/local/osquery_extensions/fleetd_tables.ext
+echo "/usr/local/osquery_extensions/fleetd_tables.ext" > /tmp/extensions.load
+```
+
+## Run osquery
+
+> The following assumes a Fleet server instance running and listening at `localhost:8080`.
+
+```sh
+sudo ENROLL_SECRET=<...> ./osquery/osqueryd \
+    --verbose=true \
+    --tls_dump=true \
+    --pidfile=/Users/luk/osqueryd/osquery.pid \
+    --database_path=/Users/luk/osqueryd/osquery.db \
+    --logger_path=/Users/luk/osqueryd/osquery_log \
+    --host_identifier=instance \
+    --tls_server_certs=/Users/luk/fleetdm/git/fleet/tools/osquery/fleet.crt \
+    --enroll_secret_env=ENROLL_SECRET \
+    --tls_hostname=localhost:8080 \
+    --enroll_tls_endpoint=/api/v1/osquery/enroll \
+    --config_plugin=tls \
+    --config_tls_endpoint=/api/v1/osquery/config \
+    --config_refresh=60 \
+    --disable_distributed=false \
+    --distributed_plugin=tls \
+    --distributed_tls_max_attempts=10 \
+    --distributed_tls_read_endpoint=/api/v1/osquery/distributed/read \
+    --distributed_tls_write_endpoint=/api/v1/osquery/distributed/write \
+    --logger_plugin=tls,filesystem \
+    --logger_tls_endpoint=/api/v1/osquery/log \
+    --disable_carver=false \
+    --carver_disable_function=false \
+    --carver_start_endpoint=/api/v1/osquery/carve/begin \
+    --carver_continue_endpoint=/api/v1/osquery/carve/block \
+    --carver_block_size=2000000 \
+    --extensions_autoload=/tmp/extensions.load
+    --allow_unsafe \
+    --enable_watchdog_debug \
+    --distributed_denylist_duration 0 \
+    --enable_extensions_watchdog 2>&1 | tee /tmp/osqueryd.log
+```
+
+## Render CPU and memory usage
+
+```sh
+./tools/loadtest/osquery/macos/gnuplot_osqueryd_cpu_memory.sh
+```
+
+> The horizontal red line is the configured CPU usage limit (hardcoded to `1200ms` in the `gnuplot_osqueryd_cpu_memory.sh`)

tools/loadtest/osquery/macos/gnuplot_osqueryd_cpu_memory.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+
+# Requirements:
+#   - ripgrep
+#   - gnuplot 
+
+# Get PID of the osquery worker process.
+osquery_pid=$(ps aux | grep -E "osqueryd\s*$" | awk {'print $2'})
+
+# Extract CPU and memory data points from logs.
+rg " (\d\d:\d\d:\d\d).* pid: $osquery_pid, cpu: (\d+)ms/\d+ms, memory: ([\d.]+)" -or '$1 $2 $3' /tmp/osqueryd.log > /tmp/osqueryd.dat
+
+# Generate gnuplot commands and render CPU and memory data points.
+cat <<EOF > gnuplot_commands.txt
+set xdata time
+set timefmt "%H:%M:%S"
+set format x "%H:%M"
+set key off
+set xtics rotate by -45
+set terminal jpeg
+
+set title 'Memory (MB)'
+set output 'osquery_worker_memory.jpg'
+plot '/tmp/osqueryd.dat' using 1:3 with linespoints linetype 7 linewidth 2 title 'Memory (MB)'
+
+set title 'CPU'
+set output 'osquery_worker_cpu.jpg'
+set yrange [0:24000]
+#
+# The calculation used by osquery for CPU limit is:
+# check_interval * number_of_physical_cores * (percent_cpu_limit / 100)
+# where default values are: check_interval=3000ms, percent_cpu_limit=10%.
+# On my Macbook with 4 physical core this gives 1200ms.
+#
+plot '/tmp/osqueryd.dat' using 1:2 with linespoints linetype 6 linewidth 2 title 'CPU', 1200 linecolor 1
+EOF
+
+gnuplot < gnuplot_commands.txt
+rm gnuplot_commands.txt
+
+open osquery_worker_cpu.jpg osquery_worker_memory.jpg