Add support for missing OS-specific osquery flags in agent options (#8743)

changes/issue-8739-support-missing-osquery-flags

@@ -0,0 +1 @@
+* Added support for more OS-specific osquery command-line flags in the agent options.

server/fleet/agent_options.go

@@ -256,6 +256,7 @@ type osqueryOptions struct {
 	OsqueryCommandLineFlagsLinux
 	OsqueryCommandLineFlagsWindows
 	OsqueryCommandLineFlagsMacOS
+	OsqueryCommandLineFlagsHidden
 }
 
 // NOTE: generate automatically with `go run ./tools/osquery-agent-options/main.go`
@@ -451,6 +452,7 @@ type osqueryCommandLineFlags struct {
 	OsqueryCommandLineFlagsLinux
 	OsqueryCommandLineFlagsWindows
 	OsqueryCommandLineFlagsMacOS
+	OsqueryCommandLineFlagsHidden
 }
 
 // the following structs are for OS-specific command-line flags supported by
@@ -470,16 +472,33 @@ type OsqueryCommandLineFlagsWindows struct {
 	EnablePowershellEventsSubscriber bool   `json:"enable_powershell_events_subscriber"`
 	EnableWindowsEventsPublisher     bool   `json:"enable_windows_events_publisher"`
 	EnableWindowsEventsSubscriber    bool   `json:"enable_windows_events_subscriber"`
+	NtfsEventPublisherDebug          bool   `json:"ntfs_event_publisher_debug"`
 	WindowsEventChannels             string `json:"windows_event_channels"`
+	UsnJournalReaderDebug            bool   `json:"usn_journal_reader_debug"`
 }
 
 type OsqueryCommandLineFlagsMacOS struct {
 	DisableEndpointsecurity    bool   `json:"disable_endpointsecurity"`
 	DisableEndpointsecurityFim bool   `json:"disable_endpointsecurity_fim"`
+	EnableKeyboardEvents       bool   `json:"enable_keyboard_events"`
+	EnableMouseEvents          bool   `json:"enable_mouse_events"`
 	EsFimMutePathLiteral       string `json:"es_fim_mute_path_literal"`
 	EsFimMutePathPrefix        string `json:"es_fim_mute_path_prefix"`
 }
 
+// those osquery flags are not OS-specific, but are also not visible using
+// osqueryd --help or select * from osquery_flags, so they can't be generated
+// by the osquery-agent-options script.
+type OsqueryCommandLineFlagsHidden struct {
+	AlsoLogToStderr       bool   `json:"alsologtostderr"`
+	EventsStreamingPlugin string `json:"events_streaming_plugin"`
+	LogBufSecs            int32  `json:"logbufsecs"`
+	LogDir                string `json:"log_dir"`
+	MaxLogSize            int32  `json:"max_log_size"`
+	MinLogLevel           int32  `json:"minloglevel"`
+	StopLoggingIfFullDisk bool   `json:"stop_logging_if_full_disk"`
+}
+
 // while ValidateJSONAgentOptions validates an entire Agent Options payload,
 // this unexported function validates a single set of options. That is, in an
 // Agent Options payload, the top-level "config" key defines a set, and each

tools/osquery-agent-options/README.md

@@ -5,3 +5,19 @@ This directory contains a script (a Go command) that generates the struct needed
 It prints the resulting Go code to stdout (the `osqueryOptions` and the `osqueryCommandLineFlags` structs), you can just copy it and insert it in the proper location in the source code to replace the existing struct (in `server/fleet/agent_options.go`).
 
 Note that the latest version of osquery should be installed for this tool to work properly (`osqueryd` and `osqueryi` must be in your $PATH).
+
+## OS-specific flags
+
+Some osquery flags are OS-specific and will not show up either with `osqueryd --help` or with the `osqueryi` query, depending on the OS you're running those on. In the code (in `server/fleet/agent_options.go`), those OS-specific flags are defined in the `OsqueryCommandLineFlags{Linux,MacOS,Windows}` structs, and the `osquery-agent-options` tool will automatically ignore from its generated struct any flag already defined as part of one of the OS-specific structs.
+
+It can be hard to even know what OS-specific flags exist, because of the fact they don't show up in `osqueryd --help` or the `osqueryi` query when not running that specific OS, and the fact that not all flags are documented in [the osquery docs](https://osquery.readthedocs.io/en/stable/). To help with this, the following bash command can be executed assuming you have the osquery repository cloned locally and checked out to the latest release version:
+
+```
+# ag is the Silver Searcher, a grep alternative, but it should work with grep too, maybe
+# with some small adjustments to the flags.
+$ ag --nofilename -o 'FLAGS_[a-z0-9_]+' | sort | uniq | cut -d _ --complement -f 1
+```
+
+This finds all flags defined in the osquery codebase (assuming all flags are built the same way). It is then possible to run a diff of this list with the list from the `osqueryi` query (e.g. `osqueryi --list 'select name from osquery_flags;'`), and the missing ones are _possibly/likely_ OS-specific. It's not an automatable task, as some judgement and manual code inspection may be necessary (some flags may be just in a test file, there may be some false-positives like `FLAGS_start` and `FLAGS_end` that are only sentinel values, the code line may be commented-out, etc.), but at least it gives a list of potential such flags.
+
+To help with the future updates to those osquery flags, the output of this shell pipe is saved to a file that is included in this directory under the name `osquery_<version>_codeflags.txt`. Please store this output for each osquery version that we process for new flags, as it allows diffing the new output with the one from the previous version and quickly know if there was any new or deleted flags.

tools/osquery-agent-options/main.go

@@ -28,6 +28,7 @@ type osqueryOptions struct { {{ range $name, $type := .Options }}
 	OsqueryCommandLineFlagsLinux
 	OsqueryCommandLineFlagsWindows
 	OsqueryCommandLineFlagsMacOS
+	OsqueryCommandLineFlagsHidden
 }
 
 // NOTE: generate automatically with ` + "`go run ./tools/osquery-agent-options/main.go`" + `
@@ -38,6 +39,7 @@ type osqueryCommandLineFlags struct { {{ range $name, $type := .Flags }}
 	OsqueryCommandLineFlagsLinux
 	OsqueryCommandLineFlagsWindows
 	OsqueryCommandLineFlagsMacOS
+	OsqueryCommandLineFlagsHidden
 }
 `))
 )
@@ -56,6 +58,7 @@ func main() {
 		fleet.OsqueryCommandLineFlagsLinux
 		fleet.OsqueryCommandLineFlagsWindows
 		fleet.OsqueryCommandLineFlagsMacOS
+		fleet.OsqueryCommandLineFlagsHidden
 	}
 	b, err := json.Marshal(allOSSpecific)
 	if err != nil {

tools/osquery-agent-options/osquery_5.5.1_codeflags.txt

@@ -0,0 +1,275 @@
+
+alarm_timeout
+allow_unsafe
+alsologtostderr
+audit_allow_accept_socket_events
+audit_allow_apparmor_events
+audit_allow_config
+audit_allow_failed_socket_events
+audit_allow_fim_events
+audit_allow_fork_process_events
+audit_allow_kill_process_events
+audit_allow_null_accept_socket_events
+audit_allow_process_events
+audit_allow_seccomp_events
+audit_allow_selinux_events
+audit_allow_sockets
+audit_allow_unix
+audit_allow_user_events
+audit_backlog_limit
+audit_backlog_wait_time
+audit_debug
+audit_fim_debug
+audit_fim_show_accesses
+audit_force_reconfigure
+audit_force_unconfigure
+audit_persist
+audit_show_partial_fim_events
+audit_show_untracked_res_warnings
+augeas_lenses
+aws_access_key_id
+aws_debug
+aws_enable_proxy
+aws_firehose_endpoint
+aws_firehose_period
+aws_firehose_stream
+aws_kinesis_disable_log_status
+aws_kinesis_endpoint
+aws_kinesis_period
+aws_kinesis_random_partition_key
+aws_kinesis_stream
+aws_profile_name
+aws_proxy_host
+aws_proxy_password
+aws_proxy_port
+aws_proxy_scheme
+aws_proxy_username
+aws_region
+aws_secret_access_key
+aws_session_token
+aws_sts_arn_role
+aws_sts_region
+aws_sts_session_name
+aws_sts_timeout
+bpf_buffer_storage_size
+bpf_perf_event_array_exp
+bpf_state_tracker_reset_time
+buffered_log_max
+carver_block_size
+carver_compression
+carver_continue_endpoint
+carver_disable_function
+carver_expiry
+carver_start_endpoint
+config_accelerated_refresh
+config_check
+config_dump
+config_enable_backup
+config_path
+config_plugin
+config_refresh
+config_tls_endpoint
+config_tls_max_attempts
+connect
+csv
+daemonize
+database_dump
+database_path
+decorations_top_level
+disable_audit
+disable_caching
+disable_carver
+disable_database
+disable_decorators
+disable_distributed
+disable_endpointsecurity
+disable_endpointsecurity_fim
+disable_enrollment
+disable_events
+disable_extensions
+disable_forensic
+disable_hash_cache
+disable_logging
+disable_memory
+disable_reenrollment
+disable_tables
+disable_watchdog
+distributed_denylist_duration
+distributed_interval
+distributed_loginfo
+distributed_plugin
+distributed_tls_max_attempts
+distributed_tls_read_endpoint
+distributed_tls_write_endpoint
+docker_socket
+enable_bpf_events
+enable_extensions_watchdog
+enable_file_events
+enable_foreign
+enable_keyboard_events
+enable_mouse_events
+enable_ntfs_event_publisher
+enable_numeric_monitoring
+enable_powershell_events_subscriber
+enable_syslog
+enable_windows_events_publisher
+enable_windows_events_subscriber
+enable_yara_string
+end
+enroll_always
+enroll_secret_env
+enroll_secret_path
+enroll_tls_endpoint
+ephemeral
+es_fim_mute_path_literal
+es_fim_mute_path_prefix
+events_enforce_denylist
+events_expiry
+events_max
+events_optimize
+events_streaming_plugin
+extension
+extension_only
+extensions_autoload
+extensions_default_index
+extensions_interval
+extensions_require
+extensions_socket
+extensions_timeout
+filename
+flag
+flagfile
+foo
+force
+groups_service_delay
+groups_service_interval
+hardware_disabled_types
+hash_cache_max
+hash_delay
+header
+host_identifier
+install
+json
+json_pretty
+keep_container_worker_open
+line
+list
+logbufsecs
+log_dir
+logger_event_type
+logger_kafka_acks
+logger_kafka_brokers
+logger_kafka_compression
+logger_kafka_topic
+logger_min_status
+logger_min_stderr
+logger_mode
+logger_numerics
+logger_path
+logger_plugin
+logger_rotate
+logger_rotate_max_files
+logger_rotate_size
+logger_snapshot_event_type
+logger_status_sync
+logger_stderr
+logger_syslog_facility
+logger_syslog_prepend_cee
+logger_tls_compress
+logger_tls_endpoint
+logger_tls_max_lines
+logger_tls_max_linesize
+logger_tls_period
+logtostderr
+lxd_socket
+max_log_size
+minloglevel
+no
+nono
+ntfs_event_publisher_debug
+nullvalue
+numeric_monitoring_filesystem_path
+numeric_monitoring_plugins
+numeric_monitoring_pre_aggregation_time
+pack
+pack_delimiter
+pack_refresh_interval
+pidfile
+planner
+plist_iterations
+port
+profile
+profile_delay
+proxy_hostname
+read_max
+regex_max_size
+registry_exceptions
+rocksdb_background_flushes
+rocksdb_buffer_blocks
+rocksdb_merge_number
+rocksdb_write_buffer
+schedule_default_interval
+schedule_epoch
+schedule_lognames
+schedule_max_drift
+schedule_reload
+schedule_reload_sql
+schedule_splay_percent
+schedule_timeout
+separator
+shell_only
+shell_only_alias
+specified_identifier
+start
+stderrthreshold
+stop_logging_if_full_disk
+syslog_events_expiry
+syslog_events_max
+syslog_pipe_path
+syslog_rate_limit
+table_delay
+table_exceptions
+test_double
+test_double_alias
+test_int32
+test_int32_alias
+test_int64
+test_int64_alias
+test_options_race_parser
+test_string
+test_string_alias
+test_string_flag
+thrift_string_size_limit
+thrift_timeout
+thrift_verbose
+tls_allow_unsafe
+tls_client_cert
+tls_client_key
+tls_disable_status_log
+tls_dump
+tls_enroll_max_attempts
+tls_enroll_max_interval
+tls_enroll_override
+tls_hostname
+tls_node_api
+tls_secret_always
+tls_server_certs
+tls_session_reuse
+tls_session_timeout
+uninstall
+users_service_delay
+users_service_interval
+usn_journal_reader_debug
+v
+verbose
+watchdog_delay
+watchdog_forced_shutdown_delay
+watchdog_latency_limit
+watchdog_level
+watchdog_max_delay
+watchdog_memory_limit
+watchdog_utilization_limit
+whatever
+windows_event_channels
+xxx
+yara_delay