From dc6bfdec57f7d6a57e2f219b45dec82d4eee02ba Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Mon, 21 Nov 2022 10:32:17 -0500 Subject: [PATCH] Add support for missing OS-specific osquery flags in agent options (#8743) --- .../issue-8739-support-missing-osquery-flags | 1 + server/fleet/agent_options.go | 19 ++ tools/osquery-agent-options/README.md | 16 + tools/osquery-agent-options/main.go | 3 + .../osquery_5.5.1_codeflags.txt | 275 ++++++++++++++++++ 5 files changed, 314 insertions(+) create mode 100644 changes/issue-8739-support-missing-osquery-flags create mode 100644 tools/osquery-agent-options/osquery_5.5.1_codeflags.txt diff --git a/changes/issue-8739-support-missing-osquery-flags b/changes/issue-8739-support-missing-osquery-flags new file mode 100644 index 0000000000..e7ac5e8db9 --- /dev/null +++ b/changes/issue-8739-support-missing-osquery-flags @@ -0,0 +1 @@ +* Added support for more OS-specific osquery command-line flags in the agent options. diff --git a/server/fleet/agent_options.go b/server/fleet/agent_options.go index b8cc21d8ab..1fb63ae89e 100644 --- a/server/fleet/agent_options.go +++ b/server/fleet/agent_options.go @@ -256,6 +256,7 @@ type osqueryOptions struct { OsqueryCommandLineFlagsLinux OsqueryCommandLineFlagsWindows OsqueryCommandLineFlagsMacOS + OsqueryCommandLineFlagsHidden } // NOTE: generate automatically with `go run ./tools/osquery-agent-options/main.go` @@ -451,6 +452,7 @@ type osqueryCommandLineFlags struct { OsqueryCommandLineFlagsLinux OsqueryCommandLineFlagsWindows OsqueryCommandLineFlagsMacOS + OsqueryCommandLineFlagsHidden } // the following structs are for OS-specific command-line flags supported by @@ -470,16 +472,33 @@ type OsqueryCommandLineFlagsWindows struct { EnablePowershellEventsSubscriber bool `json:"enable_powershell_events_subscriber"` EnableWindowsEventsPublisher bool `json:"enable_windows_events_publisher"` EnableWindowsEventsSubscriber bool `json:"enable_windows_events_subscriber"` + NtfsEventPublisherDebug bool `json:"ntfs_event_publisher_debug"` WindowsEventChannels string `json:"windows_event_channels"` + UsnJournalReaderDebug bool `json:"usn_journal_reader_debug"` } type OsqueryCommandLineFlagsMacOS struct { DisableEndpointsecurity bool `json:"disable_endpointsecurity"` DisableEndpointsecurityFim bool `json:"disable_endpointsecurity_fim"` + EnableKeyboardEvents bool `json:"enable_keyboard_events"` + EnableMouseEvents bool `json:"enable_mouse_events"` EsFimMutePathLiteral string `json:"es_fim_mute_path_literal"` EsFimMutePathPrefix string `json:"es_fim_mute_path_prefix"` } +// those osquery flags are not OS-specific, but are also not visible using +// osqueryd --help or select * from osquery_flags, so they can't be generated +// by the osquery-agent-options script. +type OsqueryCommandLineFlagsHidden struct { + AlsoLogToStderr bool `json:"alsologtostderr"` + EventsStreamingPlugin string `json:"events_streaming_plugin"` + LogBufSecs int32 `json:"logbufsecs"` + LogDir string `json:"log_dir"` + MaxLogSize int32 `json:"max_log_size"` + MinLogLevel int32 `json:"minloglevel"` + StopLoggingIfFullDisk bool `json:"stop_logging_if_full_disk"` +} + // while ValidateJSONAgentOptions validates an entire Agent Options payload, // this unexported function validates a single set of options. That is, in an // Agent Options payload, the top-level "config" key defines a set, and each diff --git a/tools/osquery-agent-options/README.md b/tools/osquery-agent-options/README.md index c7bab07205..a0b200a091 100644 --- a/tools/osquery-agent-options/README.md +++ b/tools/osquery-agent-options/README.md @@ -5,3 +5,19 @@ This directory contains a script (a Go command) that generates the struct needed It prints the resulting Go code to stdout (the `osqueryOptions` and the `osqueryCommandLineFlags` structs), you can just copy it and insert it in the proper location in the source code to replace the existing struct (in `server/fleet/agent_options.go`). Note that the latest version of osquery should be installed for this tool to work properly (`osqueryd` and `osqueryi` must be in your $PATH). + +## OS-specific flags + +Some osquery flags are OS-specific and will not show up either with `osqueryd --help` or with the `osqueryi` query, depending on the OS you're running those on. In the code (in `server/fleet/agent_options.go`), those OS-specific flags are defined in the `OsqueryCommandLineFlags{Linux,MacOS,Windows}` structs, and the `osquery-agent-options` tool will automatically ignore from its generated struct any flag already defined as part of one of the OS-specific structs. + +It can be hard to even know what OS-specific flags exist, because of the fact they don't show up in `osqueryd --help` or the `osqueryi` query when not running that specific OS, and the fact that not all flags are documented in [the osquery docs](https://osquery.readthedocs.io/en/stable/). To help with this, the following bash command can be executed assuming you have the osquery repository cloned locally and checked out to the latest release version: + +``` +# ag is the Silver Searcher, a grep alternative, but it should work with grep too, maybe +# with some small adjustments to the flags. +$ ag --nofilename -o 'FLAGS_[a-z0-9_]+' | sort | uniq | cut -d _ --complement -f 1 +``` + +This finds all flags defined in the osquery codebase (assuming all flags are built the same way). It is then possible to run a diff of this list with the list from the `osqueryi` query (e.g. `osqueryi --list 'select name from osquery_flags;'`), and the missing ones are _possibly/likely_ OS-specific. It's not an automatable task, as some judgement and manual code inspection may be necessary (some flags may be just in a test file, there may be some false-positives like `FLAGS_start` and `FLAGS_end` that are only sentinel values, the code line may be commented-out, etc.), but at least it gives a list of potential such flags. + +To help with the future updates to those osquery flags, the output of this shell pipe is saved to a file that is included in this directory under the name `osquery__codeflags.txt`. Please store this output for each osquery version that we process for new flags, as it allows diffing the new output with the one from the previous version and quickly know if there was any new or deleted flags. diff --git a/tools/osquery-agent-options/main.go b/tools/osquery-agent-options/main.go index be9e118bd3..bc644b56ad 100644 --- a/tools/osquery-agent-options/main.go +++ b/tools/osquery-agent-options/main.go @@ -28,6 +28,7 @@ type osqueryOptions struct { {{ range $name, $type := .Options }} OsqueryCommandLineFlagsLinux OsqueryCommandLineFlagsWindows OsqueryCommandLineFlagsMacOS + OsqueryCommandLineFlagsHidden } // NOTE: generate automatically with ` + "`go run ./tools/osquery-agent-options/main.go`" + ` @@ -38,6 +39,7 @@ type osqueryCommandLineFlags struct { {{ range $name, $type := .Flags }} OsqueryCommandLineFlagsLinux OsqueryCommandLineFlagsWindows OsqueryCommandLineFlagsMacOS + OsqueryCommandLineFlagsHidden } `)) ) @@ -56,6 +58,7 @@ func main() { fleet.OsqueryCommandLineFlagsLinux fleet.OsqueryCommandLineFlagsWindows fleet.OsqueryCommandLineFlagsMacOS + fleet.OsqueryCommandLineFlagsHidden } b, err := json.Marshal(allOSSpecific) if err != nil { diff --git a/tools/osquery-agent-options/osquery_5.5.1_codeflags.txt b/tools/osquery-agent-options/osquery_5.5.1_codeflags.txt new file mode 100644 index 0000000000..3fb40ef543 --- /dev/null +++ b/tools/osquery-agent-options/osquery_5.5.1_codeflags.txt @@ -0,0 +1,275 @@ + +alarm_timeout +allow_unsafe +alsologtostderr +audit_allow_accept_socket_events +audit_allow_apparmor_events +audit_allow_config +audit_allow_failed_socket_events +audit_allow_fim_events +audit_allow_fork_process_events +audit_allow_kill_process_events +audit_allow_null_accept_socket_events +audit_allow_process_events +audit_allow_seccomp_events +audit_allow_selinux_events +audit_allow_sockets +audit_allow_unix +audit_allow_user_events +audit_backlog_limit +audit_backlog_wait_time +audit_debug +audit_fim_debug +audit_fim_show_accesses +audit_force_reconfigure +audit_force_unconfigure +audit_persist +audit_show_partial_fim_events +audit_show_untracked_res_warnings +augeas_lenses +aws_access_key_id +aws_debug +aws_enable_proxy +aws_firehose_endpoint +aws_firehose_period +aws_firehose_stream +aws_kinesis_disable_log_status +aws_kinesis_endpoint +aws_kinesis_period +aws_kinesis_random_partition_key +aws_kinesis_stream +aws_profile_name +aws_proxy_host +aws_proxy_password +aws_proxy_port +aws_proxy_scheme +aws_proxy_username +aws_region +aws_secret_access_key +aws_session_token +aws_sts_arn_role +aws_sts_region +aws_sts_session_name +aws_sts_timeout +bpf_buffer_storage_size +bpf_perf_event_array_exp +bpf_state_tracker_reset_time +buffered_log_max +carver_block_size +carver_compression +carver_continue_endpoint +carver_disable_function +carver_expiry +carver_start_endpoint +config_accelerated_refresh +config_check +config_dump +config_enable_backup +config_path +config_plugin +config_refresh +config_tls_endpoint +config_tls_max_attempts +connect +csv +daemonize +database_dump +database_path +decorations_top_level +disable_audit +disable_caching +disable_carver +disable_database +disable_decorators +disable_distributed +disable_endpointsecurity +disable_endpointsecurity_fim +disable_enrollment +disable_events +disable_extensions +disable_forensic +disable_hash_cache +disable_logging +disable_memory +disable_reenrollment +disable_tables +disable_watchdog +distributed_denylist_duration +distributed_interval +distributed_loginfo +distributed_plugin +distributed_tls_max_attempts +distributed_tls_read_endpoint +distributed_tls_write_endpoint +docker_socket +enable_bpf_events +enable_extensions_watchdog +enable_file_events +enable_foreign +enable_keyboard_events +enable_mouse_events +enable_ntfs_event_publisher +enable_numeric_monitoring +enable_powershell_events_subscriber +enable_syslog +enable_windows_events_publisher +enable_windows_events_subscriber +enable_yara_string +end +enroll_always +enroll_secret_env +enroll_secret_path +enroll_tls_endpoint +ephemeral +es_fim_mute_path_literal +es_fim_mute_path_prefix +events_enforce_denylist +events_expiry +events_max +events_optimize +events_streaming_plugin +extension +extension_only +extensions_autoload +extensions_default_index +extensions_interval +extensions_require +extensions_socket +extensions_timeout +filename +flag +flagfile +foo +force +groups_service_delay +groups_service_interval +hardware_disabled_types +hash_cache_max +hash_delay +header +host_identifier +install +json +json_pretty +keep_container_worker_open +line +list +logbufsecs +log_dir +logger_event_type +logger_kafka_acks +logger_kafka_brokers +logger_kafka_compression +logger_kafka_topic +logger_min_status +logger_min_stderr +logger_mode +logger_numerics +logger_path +logger_plugin +logger_rotate +logger_rotate_max_files +logger_rotate_size +logger_snapshot_event_type +logger_status_sync +logger_stderr +logger_syslog_facility +logger_syslog_prepend_cee +logger_tls_compress +logger_tls_endpoint +logger_tls_max_lines +logger_tls_max_linesize +logger_tls_period +logtostderr +lxd_socket +max_log_size +minloglevel +no +nono +ntfs_event_publisher_debug +nullvalue +numeric_monitoring_filesystem_path +numeric_monitoring_plugins +numeric_monitoring_pre_aggregation_time +pack +pack_delimiter +pack_refresh_interval +pidfile +planner +plist_iterations +port +profile +profile_delay +proxy_hostname +read_max +regex_max_size +registry_exceptions +rocksdb_background_flushes +rocksdb_buffer_blocks +rocksdb_merge_number +rocksdb_write_buffer +schedule_default_interval +schedule_epoch +schedule_lognames +schedule_max_drift +schedule_reload +schedule_reload_sql +schedule_splay_percent +schedule_timeout +separator +shell_only +shell_only_alias +specified_identifier +start +stderrthreshold +stop_logging_if_full_disk +syslog_events_expiry +syslog_events_max +syslog_pipe_path +syslog_rate_limit +table_delay +table_exceptions +test_double +test_double_alias +test_int32 +test_int32_alias +test_int64 +test_int64_alias +test_options_race_parser +test_string +test_string_alias +test_string_flag +thrift_string_size_limit +thrift_timeout +thrift_verbose +tls_allow_unsafe +tls_client_cert +tls_client_key +tls_disable_status_log +tls_dump +tls_enroll_max_attempts +tls_enroll_max_interval +tls_enroll_override +tls_hostname +tls_node_api +tls_secret_always +tls_server_certs +tls_session_reuse +tls_session_timeout +uninstall +users_service_delay +users_service_interval +usn_journal_reader_debug +v +verbose +watchdog_delay +watchdog_forced_shutdown_delay +watchdog_latency_limit +watchdog_level +watchdog_max_delay +watchdog_memory_limit +watchdog_utilization_limit +whatever +windows_event_channels +xxx +yara_delay