From b757e447bc7de134de0393f04cbd04b921850c0c Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Thu, 16 Feb 2023 17:16:40 -0300 Subject: [PATCH] Fix private IP ingestion in `network_interface_unix` and `network_interface_windows`. (#9884) #8924 This is reproduced in dogfood for `dogfood-centos-box` and `dogfood-ubuntu-box` where their "Private IP" is also their "Public IP". Given that these hosts have their "Primary IP" configured to be their "Public IP" alongside their "Private IP", the `network_interface_unix` and `network_interface_windows` queries are now changed to ingest only private IPs for the "Private IP" field. - [X] Changes file added for user-visible changes in `changes/` or `orbit/changes/`. See [Changes files](https://fleetdm.com/docs/contributing/committing-changes#changes-files) for more information. - ~[ ] Documented any API changes (docs/Using-Fleet/REST-API.md or docs/Contributing/API-for-contributors.md)~ - ~[ ] Documented any permissions changes~ - ~[ ] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements)~ - ~[ ] Added support on fleet's osquery simulator `cmd/osquery-perf` for new osquery data ingestion features.~ - ~[ ] Added/updated tests~ - [X] Manual QA for all new/changed functionality - ~For Orbit and Fleet Desktop changes:~ - ~[ ] Manual QA must be performed in the three main OSs, macOS, Windows and Linux.~ - ~[ ] Auto-update manual QA, from released version of component to new version (see [tools/tuf/test](../tools/tuf/test/README.md)).~ --- changes/fix-private-ip-ingest-query | 1 + docs/Using-Fleet/Detail-Queries-Summary.md | 78 ++++++++++++++------ server/service/osquery_utils/queries.go | 85 +++++++++++++--------- 3 files changed, 104 insertions(+), 60 deletions(-) create mode 100644 changes/fix-private-ip-ingest-query diff --git a/changes/fix-private-ip-ingest-query b/changes/fix-private-ip-ingest-query new file mode 100644 index 0000000000..ed7ab2552f --- /dev/null +++ b/changes/fix-private-ip-ingest-query @@ -0,0 +1 @@ +* Fix `network_interface_unix` and `network_interface_windows` to ingest "Private IPs" only (filter out "Public IPs"). diff --git a/docs/Using-Fleet/Detail-Queries-Summary.md b/docs/Using-Fleet/Detail-Queries-Summary.md index a5fcd91dde..976ab0f390 100644 --- a/docs/Using-Fleet/Detail-Queries-Summary.md +++ b/docs/Using-Fleet/Detail-Queries-Summary.md @@ -171,21 +171,36 @@ select version, errors, warnings from munki_info; - Query: ```sql -select +SELECT ia.address, id.mac -from +FROM interface_addresses ia - join interface_details id on id.interface = ia.interface - join routes r on r.interface = ia.interface -where - r.destination = '0.0.0.0' - and r.netmask = 0 - and r.type = 'gateway' - and instr(ia.address, '.') > 0 -order by - r.metric asc -limit 1 + JOIN interface_details id ON id.interface = ia.interface + -- On Unix ia.interface is the name of the interface, + -- whereas on Windows ia.interface is the IP of the interface. + JOIN routes r ON r.interface = ia.interface +WHERE + -- Destination 0.0.0.0/0 is the default route on route tables. + r.destination = '0.0.0.0' AND r.netmask = 0 + -- Type of route is "gateway" for Unix, "remote" for Windows. + AND r.type = 'gateway' + -- We are only interested on private IPs (some devices have their Public IP as Primary IP too). + AND ( + -- Private IPv4 addresses. + inet_aton(ia.address) IS NOT NULL AND ( + split(ia.address, '.', 0) = '10' + OR (split(ia.address, '.', 0) = '172' AND (CAST(split(ia.address, '.', 1) AS INTEGER) & 0xf0) = 16) + OR (split(ia.address, '.', 0) = '192' AND split(ia.address, '.', 1) = '168') + ) + -- Private IPv6 addresses start with 'fc' or 'fd'. + OR (inet_aton(ia.address) IS NULL AND regex_match(lower(ia.address), '^f[cd][0-9a-f][0-9a-f]:[0-9a-f:]+', 0) IS NOT NULL) + ) +ORDER BY + r.metric ASC, + -- Prefer IPv4 addresses over IPv6 addresses if their route have the same metric. + inet_aton(ia.address) IS NOT NULL DESC +LIMIT 1; ``` ## network_interface_windows @@ -195,21 +210,36 @@ limit 1 - Query: ```sql -select +SELECT ia.address, id.mac -from +FROM interface_addresses ia - join interface_details id on id.interface = ia.interface - join routes r on r.interface = ia.address -where - r.destination = '0.0.0.0' - and r.netmask = 0 - and r.type = 'remote' - and instr(ia.address, '.') > 0 -order by - r.metric asc -limit 1 + JOIN interface_details id ON id.interface = ia.interface + -- On Unix ia.interface is the name of the interface, + -- whereas on Windows ia.interface is the IP of the interface. + JOIN routes r ON r.interface = ia.address +WHERE + -- Destination 0.0.0.0/0 is the default route on route tables. + r.destination = '0.0.0.0' AND r.netmask = 0 + -- Type of route is "gateway" for Unix, "remote" for Windows. + AND r.type = 'remote' + -- We are only interested on private IPs (some devices have their Public IP as Primary IP too). + AND ( + -- Private IPv4 addresses. + inet_aton(ia.address) IS NOT NULL AND ( + split(ia.address, '.', 0) = '10' + OR (split(ia.address, '.', 0) = '172' AND (CAST(split(ia.address, '.', 1) AS INTEGER) & 0xf0) = 16) + OR (split(ia.address, '.', 0) = '192' AND split(ia.address, '.', 1) = '168') + ) + -- Private IPv6 addresses start with 'fc' or 'fd'. + OR (inet_aton(ia.address) IS NULL AND regex_match(lower(ia.address), '^f[cd][0-9a-f][0-9a-f]:[0-9a-f:]+', 0) IS NOT NULL) + ) +ORDER BY + r.metric ASC, + -- Prefer IPv4 addresses over IPv6 addresses if their route have the same metric. + inet_aton(ia.address) IS NOT NULL DESC +LIMIT 1; ``` ## orbit_info diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index ff16a12764..6fadea0717 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -56,6 +56,46 @@ func (q *DetailQuery) RunsForPlatform(platform string) bool { return false } +// networkInterfaceQuery is the query to use to ingest a host's "Primary IP" and "Primary MAC". +// +// "Primary IP"/"Primary MAC" is the IP/MAC of the interface the system uses when it originates traffic to the default route. +// +// The following was used to determine private IPs: +// https://cs.opensource.google/go/go/+/refs/tags/go1.20.1:src/net/ip.go;l=131-148;drc=c53390b078b4d3b18e3aca8970d4b31d4d82cce1 +// +// NOTE: We cannot use `in_cidr_block` because it's available since osquery 5.3.0, so we use +// rudimentary split and string matching for IPv4 and and regex_match for IPv6. +const networkInterfaceQuery = `SELECT + ia.address, + id.mac +FROM + interface_addresses ia + JOIN interface_details id ON id.interface = ia.interface + -- On Unix ia.interface is the name of the interface, + -- whereas on Windows ia.interface is the IP of the interface. + JOIN routes r ON %s +WHERE + -- Destination 0.0.0.0/0 is the default route on route tables. + r.destination = '0.0.0.0' AND r.netmask = 0 + -- Type of route is "gateway" for Unix, "remote" for Windows. + AND r.type = '%s' + -- We are only interested on private IPs (some devices have their Public IP as Primary IP too). + AND ( + -- Private IPv4 addresses. + inet_aton(ia.address) IS NOT NULL AND ( + split(ia.address, '.', 0) = '10' + OR (split(ia.address, '.', 0) = '172' AND (CAST(split(ia.address, '.', 1) AS INTEGER) & 0xf0) = 16) + OR (split(ia.address, '.', 0) = '192' AND split(ia.address, '.', 1) = '168') + ) + -- Private IPv6 addresses start with 'fc' or 'fd'. + OR (inet_aton(ia.address) IS NULL AND regex_match(lower(ia.address), '^f[cd][0-9a-f][0-9a-f]:[0-9a-f:]+', 0) IS NOT NULL) + ) +ORDER BY + r.metric ASC, + -- Prefer IPv4 addresses over IPv6 addresses if their route have the same metric. + inet_aton(ia.address) IS NOT NULL DESC +LIMIT 1;` + // hostDetailQueries defines the detail queries that should be run on the host, as // well as how the results of those queries should be ingested into the // fleet.Host data model (via IngestFunc). @@ -63,44 +103,12 @@ func (q *DetailQuery) RunsForPlatform(platform string) bool { // This map should not be modified at runtime. var hostDetailQueries = map[string]DetailQuery{ "network_interface_unix": { - Query: ` -select - ia.address, - id.mac -from - interface_addresses ia - join interface_details id on id.interface = ia.interface - join routes r on r.interface = ia.interface -where - r.destination = '0.0.0.0' - and r.netmask = 0 - and r.type = 'gateway' - and instr(ia.address, '.') > 0 -order by - r.metric asc -limit 1 -`, + Query: fmt.Sprintf(networkInterfaceQuery, "r.interface = ia.interface", "gateway"), Platforms: append(fleet.HostLinuxOSs, "darwin"), IngestFunc: ingestNetworkInterface, }, "network_interface_windows": { - Query: ` -select - ia.address, - id.mac -from - interface_addresses ia - join interface_details id on id.interface = ia.interface - join routes r on r.interface = ia.address -where - r.destination = '0.0.0.0' - and r.netmask = 0 - and r.type = 'remote' - and instr(ia.address, '.') > 0 -order by - r.metric asc -limit 1 -`, + Query: fmt.Sprintf(networkInterfaceQuery, "r.interface = ia.address", "remote"), Platforms: []string{"windows"}, IngestFunc: ingestNetworkInterface, }, @@ -334,8 +342,13 @@ FROM logical_drives WHERE file_system = 'NTFS' LIMIT 1;`, func ingestNetworkInterface(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error { if len(rows) != 1 { - logger.Log("component", "service", "method", "IngestFunc", "err", - fmt.Sprintf("detail_query_network_interface expected single result, got %d", len(rows))) + logger.Log( + "component", "service", + "method", "IngestFunc", + "host", host.Hostname, + "platform", host.Platform, + "err", fmt.Sprintf("detail_query_network_interface expected single result, got %d", len(rows)), + ) return nil }