Allow custom osquery database on fleetd (#16554)

#16014

- [X] Changes file added for user-visible changes in `changes/` or
`orbit/changes/`.
- [x] Manual QA for all new/changed functionality
  - For Orbit and Fleet Desktop changes:
- [x] Manual QA must be performed in the three main OSs, macOS, Windows
and Linux.
- [x] Auto-update manual QA, from released version of component to new
version (see [tools/tuf/test](../tools/tuf/test/README.md)).

changes/16014-add-osquery-db-flag-to-fleetd

@@ -0,0 +1 @@
+* Add `--osquery-db` flag to `fleetctl package` command to configure a custom directory for osquery's database (`fleetctl package --osquery-db=/path/to/osquery.db`).

cmd/fleetctl/package.go

@@ -8,10 +8,12 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 
 	eefleetctl "github.com/fleetdm/fleet/v4/ee/fleetctl"
 	"github.com/fleetdm/fleet/v4/orbit/pkg/packaging"
+	"github.com/fleetdm/fleet/v4/pkg/filepath_windows"
 	"github.com/fleetdm/fleet/v4/server/fleet"
 	"github.com/rs/zerolog"
 	zlog "github.com/rs/zerolog/log"
@@ -240,6 +242,12 @@ func packageCommand() *cli.Command {
 				EnvVars:     []string{"FLEETCTL_DISABLE_KEYSTORE"},
 				Destination: &opt.DisableKeystore,
 			},
+			&cli.StringFlag{
+				Name:        "osquery-db",
+				Usage:       "Sets a custom osquery database directory, it must be an absolute path (requires orbit >= v1.22.0)",
+				EnvVars:     []string{"FLEETCTL_OSQUERY_DB"},
+				Destination: &opt.OsqueryDB,
+			},
 		},
 		Action: func(c *cli.Context) error {
 			if opt.FleetURL != "" || opt.EnrollSecret != "" {
@@ -280,6 +288,10 @@ func packageCommand() *cli.Command {
 				}
 			}
 
+			if opt.OsqueryDB != "" && !isAbsolutePath(opt.OsqueryDB, c.String("type")) {
+				return fmt.Errorf("--osquery-db must be an absolute path: %q", opt.OsqueryDB)
+			}
+
 			if runtime.GOOS == "windows" && c.String("type") != "msi" {
 				return errors.New("Windows can only build MSI packages.")
 			}
@@ -372,3 +384,13 @@ func checkPEMCertificate(path string) error {
 	}
 	return nil
 }
+
+// isAbsolutePath returns whether a path is absolute.
+// It does not make use of filepath.IsAbs to support
+// checking Windows paths from Go code running in unix.
+func isAbsolutePath(path, pkgType string) bool {
+	if pkgType == "msi" {
+		return filepath_windows.IsAbs(path)
+	}
+	return strings.HasPrefix(path, "/") // this is the unix implementation of filepath.IsAbs
+}

orbit/changes/16014-add-osquery-db-flag-to-fleetd

@@ -0,0 +1 @@
+* Allow configuring a custom osquery database directory (`ORBIT_OSQUERY_DB` environment variable or `--osquery-db` flag).

orbit/cmd/orbit/orbit.go

@@ -197,6 +197,11 @@ func main() {
 			Usage:   "Disables the use of the keychain on macOS and Credentials Manager on Windows",
 			EnvVars: []string{"ORBIT_DISABLE_KEYSTORE"},
 		},
+		&cli.StringFlag{
+			Name:    "osquery-db",
+			Usage:   "Sets a custom osquery database directory, it must be an absolute path",
+			EnvVars: []string{"ORBIT_OSQUERY_DB"},
+		},
 	}
 	app.Before = func(c *cli.Context) error {
 		// handle old installations, which had default root dir set to /var/lib/orbit
@@ -270,6 +275,10 @@ func main() {
 			return errors.New("insecure and update-tls-certificate may not be specified together")
 		}
 
+		if odb := c.String("osquery-db"); odb != "" && !filepath.IsAbs(odb) {
+			return fmt.Errorf("the osquery database must be an absolute path: %q", odb)
+		}
+
 		enrollSecretPath := c.String("enroll-secret-path")
 		if enrollSecretPath != "" {
 			if c.String("enroll-secret") != "" {
@@ -588,7 +597,12 @@ func main() {
 			log.Debug().Str("processes", fmt.Sprintf("%+v", killedProcesses)).Msg("existing osqueryd processes killed")
 		}
 
-		osqueryHostInfo, err := getHostInfo(osquerydPath, filepath.Join(c.String("root-dir"), "osquery.db"))
+		osqueryDB := filepath.Join(c.String("root-dir"), "osquery.db")
+		if odb := c.String("osquery-db"); odb != "" {
+			osqueryDB = odb
+		}
+
+		osqueryHostInfo, err := getHostInfo(osquerydPath, osqueryDB)
 		if err != nil {
 			return fmt.Errorf("get UUID: %w", err)
 		}
@@ -614,11 +628,16 @@ func main() {
 		}
 
 		var (
-			options              []osquery.Option
+			options []osquery.Option
+			// optionsAfterFlagfile is populated with options that will be set after the '--flagfile' argument
+			// to not allow users to change their values on their flagfiles.
 			optionsAfterFlagfile []osquery.Option
 		)
-		options = append(options, osquery.WithDataPath(c.String("root-dir")))
+		options = append(options, osquery.WithDataPath(c.String("root-dir"), ""))
 		options = append(options, osquery.WithLogPath(filepath.Join(c.String("root-dir"), "osquery_log")))
+		optionsAfterFlagfile = append(optionsAfterFlagfile, osquery.WithFlags(
+			[]string{"--database_path", osqueryDB},
+		))
 
 		if logFile != nil {
 			// If set, redirect osqueryd's stderr to the logFile.
@@ -1430,6 +1449,10 @@ type osqueryHostInfo struct {
 
 // getHostInfo retrieves system information about the host by shelling out to `osqueryd -S` and performing a `SELECT` query.
 func getHostInfo(osqueryPath string, osqueryDBPath string) (*osqueryHostInfo, error) {
+	// Make sure parent directory exists (`osqueryd -S` doesn't create the parent directories).
+	if err := os.MkdirAll(filepath.Dir(osqueryDBPath), constant.DefaultDirMode); err != nil {
+		return nil, err
+	}
 	const systemQuery = "SELECT si.uuid, si.hardware_serial, si.hostname, os.platform, oi.instance_id FROM system_info si, os_version os, osquery_info oi"
 	args := []string{
 		"-S",

orbit/cmd/orbit/shell.go

@@ -78,14 +78,24 @@ var shellCommand = &cli.Command{
 
 		var g run.Group
 
+		// We use a suffix on the extension path on Windows
+		// because there was an issue when the osqueryd instance ran through
+		// `orbit shell` attempted to register the same named pipe name used by
+		// the osqueryd instance launched by orbit service.
 		extensionPathPostfix := ""
 		if runtime.GOOS == "windows" {
 			extensionPathPostfix = "-" + uuid.New().String()
 		}
 
+		// We use a different path from the orbit daemon
+		// to avoid issues with concurrent accesses to files/databases
+		// (RocksDB lock and/or Windows file locking).
+		dataPath := filepath.Join(c.String("root-dir"), "shell")
+		osqueryDB := filepath.Join(dataPath, "osquery.db")
 		opts := []osquery.Option{
 			osquery.WithShell(),
-			osquery.WithDataPathAndExtensionPathPostfix(filepath.Join(c.String("root-dir"), "shell"), extensionPathPostfix),
+			osquery.WithDataPath(dataPath, extensionPathPostfix),
+			osquery.WithFlags([]string{"--database_path", osqueryDB}),
 		}
 
 		// Detect if the additional arguments have a positional argument.

orbit/pkg/osquery/osquery.go

@@ -102,34 +102,19 @@ func WithShell() func(*Runner) error {
 	}
 }
 
-func WithDataPath(path string) Option {
+// WithDataPath configures the dataPath in the *Runner and
+// sets the --pidfile and --extensions_socket paths
+// to the osqueryd invocation.
+func WithDataPath(dataPath, extensionPathPostfix string) Option {
 	return func(r *Runner) error {
-		r.dataPath = path
+		r.dataPath = dataPath
 
-		if err := secure.MkdirAll(path, constant.DefaultDirMode); err != nil {
+		if err := secure.MkdirAll(dataPath, constant.DefaultDirMode); err != nil {
 			return fmt.Errorf("initialize osquery data path: %w", err)
 		}
 
 		r.cmd.Args = append(r.cmd.Args,
-			"--pidfile="+filepath.Join(path, constant.OsqueryPidfile),
-			"--database_path="+filepath.Join(path, "osquery.db"),
-			"--extensions_socket="+r.ExtensionSocketPath(),
-		)
-		return nil
-	}
-}
-
-func WithDataPathAndExtensionPathPostfix(path string, extensionPathPostfix string) Option {
-	return func(r *Runner) error {
-		r.dataPath = path
-
-		if err := secure.MkdirAll(path, constant.DefaultDirMode); err != nil {
-			return fmt.Errorf("initialize osquery data path: %w", err)
-		}
-
-		r.cmd.Args = append(r.cmd.Args,
-			"--pidfile="+filepath.Join(path, constant.OsqueryPidfile),
-			"--database_path="+filepath.Join(path, "osquery.db"),
+			"--pidfile="+filepath.Join(dataPath, constant.OsqueryPidfile),
 			"--extensions_socket="+r.ExtensionSocketPath()+extensionPathPostfix,
 		)
 		return nil

orbit/pkg/packaging/linux_shared.go

@@ -294,6 +294,7 @@ ORBIT_FLEET_DESKTOP_ALTERNATIVE_BROWSER_HOST={{ .FleetDesktopAlternativeBrowserH
 {{ if .Debug }}ORBIT_DEBUG=true{{ end }}
 {{ if .EnableScripts }}ORBIT_ENABLE_SCRIPTS=true{{ end }}
 {{ if and (ne .HostIdentifier "") (ne .HostIdentifier "uuid") }}ORBIT_HOST_IDENTIFIER={{.HostIdentifier}}{{ end }}
+{{ if .OsqueryDB }}ORBIT_OSQUERY_DB={{.OsqueryDB}}{{ end }}
 `))
 
 func writeEnvFile(opt Options, rootPath string) error {

orbit/pkg/packaging/macos_templates.go

@@ -161,6 +161,10 @@ var macosLaunchdTemplate = template.Must(template.New("").Option("missingkey=err
 		<key>ORBIT_DISABLE_KEYSTORE</key>
 		<string>true</string>
 		{{- end }}
+		{{- if .OsqueryDB }}
+		<key>ORBIT_OSQUERY_DB</key>
+		<string>{{ .OsqueryDB }}</string>
+		{{- end }}
 	</dict>
 	<key>KeepAlive</key>
 	<true/>

orbit/pkg/packaging/packaging.go

@@ -122,6 +122,9 @@ type Options struct {
 	EndUserEmail string
 	// DisableKeystore disables the use of the keychain on macOS and Credentials Manager on Windows
 	DisableKeystore bool
+	// OsqueryDB is the directory to use for the osquery database.
+	// If not set, then the default is `$ORBIT_ROOT_DIR/osquery.db`.
+	OsqueryDB string
 }
 
 func initializeTempDir() (string, error) {

orbit/pkg/packaging/windows_templates.go

@@ -99,7 +99,7 @@ var windowsWixTemplate = template.Must(template.New("").Option("missingkey=error
                   Start="auto"
                   Type="ownProcess"
                   Description="This service runs Fleet's osquery runtime and autoupdater (Orbit)."
-                  Arguments='--root-dir "[ORBITROOT]." --log-file "[System64Folder]config\systemprofile\AppData\Local\FleetDM\Orbit\Logs\orbit-osquery.log" --fleet-url "[FLEET_URL]"{{ if .FleetCertificate }} --fleet-certificate "[ORBITROOT]fleet.pem"{{ end }}{{ if .EnrollSecret }} --enroll-secret-path "[ORBITROOT]secret.txt"{{ end }}{{if .Insecure }} --insecure{{ end }}{{ if .Debug }} --debug{{ end }}{{ if .UpdateURL }} --update-url "{{ .UpdateURL }}"{{ end }}{{ if .UpdateTLSServerCertificate }} --update-tls-certificate "[ORBITROOT]update.pem"{{ end }}{{ if .DisableUpdates }} --disable-updates{{ end }}{{ if .Desktop }} --fleet-desktop --desktop-channel {{ .DesktopChannel }}{{ if .FleetDesktopAlternativeBrowserHost }} --fleet-desktop-alternative-browser-host {{ .FleetDesktopAlternativeBrowserHost }}{{ end }}{{ end }} --orbit-channel "{{ .OrbitChannel }}" --osqueryd-channel "{{ .OsquerydChannel }}" {{ if .EnableScripts }} --enable-scripts{{ end }}{{ if and (ne .HostIdentifier "") (ne .HostIdentifier "uuid") }}--host-identifier={{ .HostIdentifier }}{{ end }}{{ if .EndUserEmail }} --end-user-email "{{ .EndUserEmail }}"{{ end }}'
+                  Arguments='--root-dir "[ORBITROOT]." --log-file "[System64Folder]config\systemprofile\AppData\Local\FleetDM\Orbit\Logs\orbit-osquery.log" --fleet-url "[FLEET_URL]"{{ if .FleetCertificate }} --fleet-certificate "[ORBITROOT]fleet.pem"{{ end }}{{ if .EnrollSecret }} --enroll-secret-path "[ORBITROOT]secret.txt"{{ end }}{{if .Insecure }} --insecure{{ end }}{{ if .Debug }} --debug{{ end }}{{ if .UpdateURL }} --update-url "{{ .UpdateURL }}"{{ end }}{{ if .UpdateTLSServerCertificate }} --update-tls-certificate "[ORBITROOT]update.pem"{{ end }}{{ if .DisableUpdates }} --disable-updates{{ end }}{{ if .Desktop }} --fleet-desktop --desktop-channel {{ .DesktopChannel }}{{ if .FleetDesktopAlternativeBrowserHost }} --fleet-desktop-alternative-browser-host {{ .FleetDesktopAlternativeBrowserHost }}{{ end }}{{ end }} --orbit-channel "{{ .OrbitChannel }}" --osqueryd-channel "{{ .OsquerydChannel }}" {{ if .EnableScripts }} --enable-scripts{{ end }}{{ if and (ne .HostIdentifier "") (ne .HostIdentifier "uuid") }}--host-identifier={{ .HostIdentifier }}{{ end }}{{ if .EndUserEmail }} --end-user-email "{{ .EndUserEmail }}"{{ end }}{{ if .OsqueryDB }} --osquery-db="{{ .OsqueryDB }}"{{ end }}'
                 >
                   <util:ServiceConfig
                     FirstFailureActionType="restart"

pkg/filepath_windows/filepath.go

@@ -0,0 +1,137 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package filepath_windows was copied from https://github.com/golang/go/blob/master/src/path/filepath/path_windows.go
+// to be able to verify Windows paths from Go code running on unix.
+package filepath_windows
+
+func isSlash(c uint8) bool {
+	return c == '\\' || c == '/'
+}
+
+// IsAbs reports whether the path is absolute.
+func IsAbs(path string) (b bool) {
+	l := volumeNameLen(path)
+	if l == 0 {
+		return false
+	}
+	// If the volume name starts with a double slash, this is an absolute path.
+	if isSlash(path[0]) && isSlash(path[1]) {
+		return true
+	}
+	path = path[l:]
+	if path == "" {
+		return false
+	}
+	return isSlash(path[0])
+}
+
+func toUpper(c byte) byte {
+	if 'a' <= c && c <= 'z' {
+		return c - ('a' - 'A')
+	}
+	return c
+}
+
+// pathHasPrefixFold tests whether the path s begins with prefix,
+// ignoring case and treating all path separators as equivalent.
+// If s is longer than prefix, then s[len(prefix)] must be a path separator.
+func pathHasPrefixFold(s, prefix string) bool {
+	if len(s) < len(prefix) {
+		return false
+	}
+	for i := 0; i < len(prefix); i++ {
+		if isSlash(prefix[i]) {
+			if !isSlash(s[i]) {
+				return false
+			}
+		} else if toUpper(prefix[i]) != toUpper(s[i]) {
+			return false
+		}
+	}
+	if len(s) > len(prefix) && !isSlash(s[len(prefix)]) {
+		return false
+	}
+	return true
+}
+
+// volumeNameLen returns length of the leading volume name on Windows.
+// It returns 0 elsewhere.
+//
+// See:
+// https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats
+// https://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html
+func volumeNameLen(path string) int {
+	switch {
+	case len(path) >= 2 && path[1] == ':':
+		// Path starts with a drive letter.
+		//
+		// Not all Windows functions necessarily enforce the requirement that
+		// drive letters be in the set A-Z, and we don't try to here.
+		//
+		// We don't handle the case of a path starting with a non-ASCII character,
+		// in which case the "drive letter" might be multiple bytes long.
+		return 2
+
+	case len(path) == 0 || !isSlash(path[0]):
+		// Path does not have a volume component.
+		return 0
+
+	case pathHasPrefixFold(path, `\\.\UNC`):
+		// We're going to treat the UNC host and share as part of the volume
+		// prefix for historical reasons, but this isn't really principled;
+		// Windows's own GetFullPathName will happily remove the first
+		// component of the path in this space, converting
+		// \\.\unc\a\b\..\c into \\.\unc\a\c.
+		return uncLen(path, len(`\\.\UNC\`))
+
+	case pathHasPrefixFold(path, `\\.`) ||
+		pathHasPrefixFold(path, `\\?`) || pathHasPrefixFold(path, `\??`):
+		// Path starts with \\.\, and is a Local Device path; or
+		// path starts with \\?\ or \??\ and is a Root Local Device path.
+		//
+		// We treat the next component after the \\.\ prefix as
+		// part of the volume name, which means Clean(`\\?\c:\`)
+		// won't remove the trailing \. (See #64028.)
+		if len(path) == 3 {
+			return 3 // exactly \\.
+		}
+		_, rest, ok := cutPath(path[4:])
+		if !ok {
+			return len(path)
+		}
+		return len(path) - len(rest) - 1
+
+	case len(path) >= 2 && isSlash(path[1]):
+		// Path starts with \\, and is a UNC path.
+		return uncLen(path, 2)
+	}
+	return 0
+}
+
+// uncLen returns the length of the volume prefix of a UNC path.
+// prefixLen is the prefix prior to the start of the UNC host;
+// for example, for "//host/share", the prefixLen is len("//")==2.
+func uncLen(path string, prefixLen int) int {
+	count := 0
+	for i := prefixLen; i < len(path); i++ {
+		if isSlash(path[i]) {
+			count++
+			if count == 2 {
+				return i
+			}
+		}
+	}
+	return len(path)
+}
+
+// cutPath slices path around the first path separator.
+func cutPath(path string) (before, after string, found bool) {
+	for i := range path {
+		if isSlash(path[i]) {
+			return path[:i], path[i+1:], true
+		}
+	}
+	return path, "", false
+}

server/fleet/agent_options.go

@@ -61,6 +61,9 @@ func ValidateJSONAgentOptions(ctx context.Context, ds Datastore, rawJSON json.Ra
 		if flags.ExtensionsAutoload != "" {
 			return fmt.Errorf(flagNotSupportedErr, "--extensions_autoload")
 		}
+		if flags.DatabasePath != "" {
+			return fmt.Errorf(flagNotSupportedErr, "--database_path")
+		}
 	}
 
 	if len(opts.UpdateChannels) > 0 {

tools/tuf/test/create_repository.sh

@@ -29,7 +29,7 @@ SWIFT_DIALOG_MACOS_APP_VERSION=2.2.1
 SWIFT_DIALOG_MACOS_APP_BUILD_VERSION=4591
 
 if [[ -z "$OSQUERY_VERSION" ]]; then
-    OSQUERY_VERSION=5.10.2
+    OSQUERY_VERSION=5.11.0
 fi
 
 mkdir -p $TUF_PATH/tmp